You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2018/11/10 20:59:04 UTC

[01/22] phoenix git commit: PHOENIX-4977 Make KillServerOnFailurePolicy a configurable option in PhoenixIndexFailurePolicy

Repository: phoenix
Updated Branches:
  refs/heads/omid2 0f2b1b8ba -> d03eadc23


PHOENIX-4977 Make KillServerOnFailurePolicy a configurable option in PhoenixIndexFailurePolicy


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a48e42e7
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a48e42e7
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a48e42e7

Branch: refs/heads/omid2
Commit: a48e42e7f639c8194db84155ce73ca9d4df25d91
Parents: 2ff316a
Author: Vincent Poon <vi...@apache.org>
Authored: Tue Oct 23 16:03:22 2018 -0700
Committer: Vincent Poon <vi...@apache.org>
Committed: Thu Oct 25 12:06:31 2018 -0700

----------------------------------------------------------------------
 .../phoenix/hbase/index/write/DelegateIndexFailurePolicy.java  | 5 ++++-
 .../org/apache/phoenix/index/PhoenixIndexFailurePolicy.java    | 6 ++++++
 .../src/main/java/org/apache/phoenix/query/QueryServices.java  | 1 +
 3 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/a48e42e7/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/DelegateIndexFailurePolicy.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/DelegateIndexFailurePolicy.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/DelegateIndexFailurePolicy.java
index a7fb7ec..caf2b38 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/DelegateIndexFailurePolicy.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/DelegateIndexFailurePolicy.java
@@ -28,7 +28,7 @@ import com.google.common.collect.Multimap;
 
 public class DelegateIndexFailurePolicy implements IndexFailurePolicy {
 
-    private final IndexFailurePolicy delegate;
+    private IndexFailurePolicy delegate;
     
     public DelegateIndexFailurePolicy(IndexFailurePolicy delegate) {
         this.delegate = delegate;
@@ -55,4 +55,7 @@ public class DelegateIndexFailurePolicy implements IndexFailurePolicy {
         delegate.stop(arg0);
     }
 
+    public void setDelegate(IndexFailurePolicy delegate) {
+        this.delegate = delegate;
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/a48e42e7/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
index 4c2013b..f0af6f6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
@@ -60,6 +60,7 @@ import org.apache.phoenix.hbase.index.table.HTableInterfaceReference;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 import org.apache.phoenix.hbase.index.write.DelegateIndexFailurePolicy;
 import org.apache.phoenix.hbase.index.write.KillServerOnFailurePolicy;
+import org.apache.phoenix.hbase.index.write.LeaveIndexActiveFailurePolicy;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.query.QueryServices;
@@ -133,6 +134,11 @@ public class PhoenixIndexFailurePolicy extends DelegateIndexFailurePolicy {
         } else {
         	throwIndexWriteFailure = Boolean.parseBoolean(value);
         }
+
+        boolean killServer = env.getConfiguration().getBoolean(QueryServices.INDEX_FAILURE_KILL_SERVER, true);
+        if (!killServer) {
+            setDelegate(new LeaveIndexActiveFailurePolicy());
+        } // else, default in constructor is KillServerOnFailurePolicy
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/phoenix/blob/a48e42e7/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index d3c8dc0..337bb05 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -154,6 +154,7 @@ public interface QueryServices extends SQLCloseable {
     public static final String INDEX_FAILURE_BLOCK_WRITE = "phoenix.index.failure.block.write";
     public static final String INDEX_FAILURE_DISABLE_INDEX = "phoenix.index.failure.disable.index";
     public static final String INDEX_FAILURE_THROW_EXCEPTION_ATTRIB = "phoenix.index.failure.throw.exception";
+    public static final String INDEX_FAILURE_KILL_SERVER = "phoenix.index.failure.unhandled.killserver";
 
     // Index will be partially re-built from index disable time stamp - following overlap time
     @Deprecated


[06/22] phoenix git commit: PHOENIX-4764 Cleanup metadata of child views for a base table that has been dropped (addendum)

Posted by ja...@apache.org.
PHOENIX-4764 Cleanup metadata of child views for a base table that has been dropped (addendum)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c936d815
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c936d815
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c936d815

Branch: refs/heads/omid2
Commit: c936d815119b73dc927a7b3bb6a2b577b4a2f91c
Parents: 6478879
Author: Kadir <ko...@salesforce.com>
Authored: Tue Oct 30 17:47:26 2018 -0700
Committer: Karan Mehta <ka...@gmail.com>
Committed: Tue Oct 30 18:33:10 2018 -0700

----------------------------------------------------------------------
 .../end2end/MigrateSystemTablesToSystemNamespaceIT.java       | 4 ++--
 .../phoenix/end2end/SystemCatalogCreationOnConnectionIT.java  | 4 ++--
 .../org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java   | 4 +++-
 .../org/apache/phoenix/coprocessor/TaskRegionObserver.java    | 7 +++++--
 .../org/apache/phoenix/query/ConnectionQueryServicesImpl.java | 2 +-
 .../src/main/java/org/apache/phoenix/query/QueryServices.java | 2 ++
 .../java/org/apache/phoenix/query/QueryServicesOptions.java   | 3 ++-
 7 files changed, 17 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/c936d815/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
index ffac4d6..b6f061e 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
@@ -62,10 +62,10 @@ public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest {
 
     private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
             "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
-            "SYSTEM.MUTEX","SYSTEM.LOG", "SYSTEM.CHILD_LINK"));
+            "SYSTEM.MUTEX","SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK"));
     private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
             Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
-                    "SYSTEM:MUTEX","SYSTEM:LOG", "SYSTEM:CHILD_LINK"));
+                    "SYSTEM:MUTEX","SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK"));
     private static final String SCHEMA_NAME = "MIGRATETEST";
     private static final String TABLE_NAME =
             SCHEMA_NAME + "." + MigrateSystemTablesToSystemNamespaceIT.class.getSimpleName().toUpperCase();

http://git-wip-us.apache.org/repos/asf/phoenix/blob/c936d815/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
index aa2d971..a1685c44 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
@@ -72,11 +72,11 @@ public class SystemCatalogCreationOnConnectionIT {
 
     private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
       "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
-      "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK"));
+      "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK"));
 
     private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
       Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
-        "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK"));
+        "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK"));
 
     private static class PhoenixSysCatCreationServices extends ConnectionQueryServicesImpl {
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/c936d815/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java
index 5c016f6..9c2b763 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.monitoring;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import org.apache.phoenix.coprocessor.TaskRegionObserver;
 import org.apache.phoenix.end2end.BaseUniqueNamesOwnClusterIT;
 import org.apache.phoenix.jdbc.PhoenixDriver;
 import org.apache.phoenix.query.QueryServices;
@@ -53,7 +54,8 @@ public class BasePhoenixMetricsIT extends BaseUniqueNamesOwnClusterIT {
 
     @BeforeClass
     public static void doSetup() throws Exception {
-        Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+        Map<String, String> props = Maps.newHashMapWithExpectedSize(3);
+        props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
         // Phoenix Global client metrics are enabled by default
         // Enable request metric collection at the driver level
         props.put(QueryServices.COLLECT_REQUEST_LEVEL_METRICS, String.valueOf(true));

http://git-wip-us.apache.org/repos/asf/phoenix/blob/c936d815/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
index ca71961..87f7414 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
@@ -68,8 +68,7 @@ public class TaskRegionObserver extends BaseRegionObserver {
     private long timeInterval = QueryServicesOptions.DEFAULT_TASK_HANDLING_INTERVAL_MS;
     private long timeMaxInterval = QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS;
     @GuardedBy("TaskRegionObserver.class")
-    // initial delay before the first task is handled
-    private static final long  initialDelay = 10000; // 10 secs
+    private long initialDelay;
 
     @Override
     public void preClose(final ObserverContext<RegionCoprocessorEnvironment> c,
@@ -88,6 +87,10 @@ public class TaskRegionObserver extends BaseRegionObserver {
                 config.getLong(
                         QueryServices.TASK_HANDLING_MAX_INTERVAL_MS_ATTRIB,
                         QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS);
+        initialDelay =
+                config.getLong(
+                        QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB,
+                        QueryServicesOptions.DEFAULT_TASK_HANDLING_INITIAL_DELAY_MS);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/phoenix/blob/c936d815/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 1381247..8c09833 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -1177,7 +1177,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                     splits, isNamespaceMapped);
 
             if (!tableExist) {
-                if (isMetaTable && !isUpgradeRequired() && (!isAutoUpgradeEnabled || isDoNotUpgradePropSet)) {
+                if (SchemaUtil.isSystemTable(physicalTableName) && !isUpgradeRequired() && (!isAutoUpgradeEnabled || isDoNotUpgradePropSet)) {
                     // Disallow creating the SYSTEM.CATALOG or SYSTEM:CATALOG HBase table
                     throw new UpgradeRequiredException();
                 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/c936d815/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index d06f07c..a8bbc22 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -325,6 +325,8 @@ public interface QueryServices extends SQLCloseable {
     public static final String TASK_HANDLING_INTERVAL_MS_ATTRIB = "phoenix.task.handling.interval.ms";
     // The maximum time for a task to stay in table SYSTEM.TASK
     public static final String TASK_HANDLING_MAX_INTERVAL_MS_ATTRIB = "phoenix.task.handling.maxInterval.ms";
+    // The initial delay before the first task from table SYSTEM.TASK is handled
+    public static final String TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB = "phoenix.task.handling.initial.delay.ms";
 
     /**
      * Get executor service used for parallel scans

http://git-wip-us.apache.org/repos/asf/phoenix/blob/c936d815/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index a6a73f7..d2f5bce 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -342,7 +342,8 @@ public class QueryServicesOptions {
 
     // default system task handling interval in milliseconds
     public static final long DEFAULT_TASK_HANDLING_INTERVAL_MS = 60*1000; // 1 min
-    public static final long DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS = 30*60*1000; // 30 minutes
+    public static final long DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS = 30*60*1000; // 30 min
+    public static final long DEFAULT_TASK_HANDLING_INITIAL_DELAY_MS = 10*1000; // 10 sec
 
     @SuppressWarnings("serial")
     public static final Set<String> DEFAULT_QUERY_SERVER_SKIP_WORDS = new HashSet<String>() {


[12/22] phoenix git commit: PHOENIX-4981 Add tests for ORDER BY, GROUP BY and salted tables using phoenix-spark

Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java
index 578a3af..792d08f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java
@@ -17,17 +17,7 @@
  */
 package org.apache.phoenix.end2end;
 
-import static org.apache.phoenix.util.TestUtil.ROW1;
-import static org.apache.phoenix.util.TestUtil.ROW2;
-import static org.apache.phoenix.util.TestUtil.ROW3;
-import static org.apache.phoenix.util.TestUtil.ROW4;
-import static org.apache.phoenix.util.TestUtil.ROW5;
-import static org.apache.phoenix.util.TestUtil.ROW6;
-import static org.apache.phoenix.util.TestUtil.ROW7;
-import static org.apache.phoenix.util.TestUtil.ROW8;
-import static org.apache.phoenix.util.TestUtil.ROW9;
 import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-import static org.apache.phoenix.util.TestUtil.assertResultSet;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -40,83 +30,10 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.Properties;
 
-import org.apache.phoenix.jdbc.PhoenixStatement;
 import org.apache.phoenix.util.PropertiesUtil;
 import org.junit.Test;
 
-
-public class OrderByIT extends ParallelStatsDisabledIT {
-
-    @Test
-    public void testMultiOrderByExpr() throws Exception {
-        String tenantId = getOrganizationId();
-        String tableName = initATableValues(tenantId, getDefaultSplits(tenantId), getUrl());
-        String query = "SELECT entity_id FROM " + tableName + " ORDER BY b_string, entity_id";
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        try {
-            PreparedStatement statement = conn.prepareStatement(query);
-            ResultSet rs = statement.executeQuery();
-            assertTrue (rs.next());
-            assertEquals(ROW1,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW4,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW7,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW2,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW5,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW8,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW3,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW6,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW9,rs.getString(1));
-
-            assertFalse(rs.next());
-        } finally {
-            conn.close();
-        }
-    }
-
-
-    @Test
-    public void testDescMultiOrderByExpr() throws Exception {
-        String tenantId = getOrganizationId();
-        String tableName = initATableValues(tenantId, getDefaultSplits(tenantId), getUrl());
-        String query = "SELECT entity_id FROM " + tableName + " ORDER BY b_string || entity_id desc";
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        try {
-            PreparedStatement statement = conn.prepareStatement(query);
-            ResultSet rs = statement.executeQuery();
-            assertTrue (rs.next());
-            assertEquals(ROW9,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW6,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW3,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW8,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW5,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW2,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW7,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW4,rs.getString(1));
-            assertTrue (rs.next());
-            assertEquals(ROW1,rs.getString(1));
-
-            assertFalse(rs.next());
-        } finally {
-            conn.close();
-        }
-    }
+public class OrderByIT extends BaseOrderByIT {
 
     @Test
     public void testOrderByWithPosition() throws Exception {
@@ -151,8 +68,8 @@ public class OrderByIT extends ParallelStatsDisabledIT {
             assertTrue(rs.next());
             assertEquals(1,rs.getInt(1));
             assertTrue(rs.next());
-            assertEquals(1,rs.getInt(1));  
-            assertFalse(rs.next());  
+            assertEquals(1,rs.getInt(1));
+            assertFalse(rs.next());
 
             query = "select a_string x, col1 y from " + tableName + " order by x";
             rs = conn.createStatement().executeQuery(query);
@@ -163,9 +80,9 @@ public class OrderByIT extends ParallelStatsDisabledIT {
             assertEquals("b",rs.getString(1));
             assertEquals(20,rs.getInt(2));
             assertTrue(rs.next());
-            assertEquals("c",rs.getString(1));  
+            assertEquals("c",rs.getString(1));
             assertEquals(30,rs.getInt(2));
-            assertFalse(rs.next());  
+            assertFalse(rs.next());
 
             query = "select * from " + tableName + " order by 2";
             rs = conn.createStatement().executeQuery(query);
@@ -173,113 +90,12 @@ public class OrderByIT extends ParallelStatsDisabledIT {
             assertEquals("b",rs.getString(1));
             assertEquals(20,rs.getInt(2));
             assertTrue(rs.next());
-            assertEquals("c",rs.getString(1));  
-            assertEquals(30,rs.getInt(2));
-            assertTrue(rs.next());
-            assertEquals("a",rs.getString(1));
-            assertEquals(40,rs.getInt(2));
-            assertFalse(rs.next());  
-        } finally {
-            conn.close();
-        }
-    }
-
-
-    @Test
-    public void testColumnFamily() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        conn.setAutoCommit(false);
-
-        try {
-            String tableName = generateUniqueName();
-            String ddl = "CREATE TABLE " + tableName +
-                    "  (a_string varchar not null, cf1.a integer, cf1.b varchar, col1 integer, cf2.c varchar, cf2.d integer, col2 integer" +
-                    "  CONSTRAINT pk PRIMARY KEY (a_string))\n";
-            createTestTable(getUrl(), ddl);
-            String dml = "UPSERT INTO " + tableName + " VALUES(?,?,?,?,?,?,?)";
-            PreparedStatement stmt = conn.prepareStatement(dml);
-            stmt.setString(1, "a");
-            stmt.setInt(2, 40);
-            stmt.setString(3, "aa");
-            stmt.setInt(4, 10);
-            stmt.setString(5, "bb");
-            stmt.setInt(6, 20);
-            stmt.setInt(7, 1);
-            stmt.execute();
-            stmt.setString(1, "c");
-            stmt.setInt(2, 30);
-            stmt.setString(3, "cc");
-            stmt.setInt(4, 50);
-            stmt.setString(5, "dd");
-            stmt.setInt(6, 60);
-            stmt.setInt(7, 3);
-            stmt.execute();
-            stmt.setString(1, "b");
-            stmt.setInt(2, 40);
-            stmt.setString(3, "bb");
-            stmt.setInt(4, 5);
-            stmt.setString(5, "aa");
-            stmt.setInt(6, 80);
-            stmt.setInt(7, 2);
-            stmt.execute();
-            conn.commit();
-
-            String query = "select * from " + tableName + " order by 2, 5";
-            ResultSet rs = conn.createStatement().executeQuery(query);
-            assertTrue(rs.next());
             assertEquals("c",rs.getString(1));
             assertEquals(30,rs.getInt(2));
-            assertEquals("cc",rs.getString(3));
-            assertEquals(50,rs.getInt(4));
-            assertEquals("dd",rs.getString(5));
-            assertEquals(60,rs.getInt(6));
-            assertEquals(3,rs.getInt(7));
-            assertTrue(rs.next());
-            assertEquals("b",rs.getString(1));  
-            assertEquals(40,rs.getInt(2));
-            assertEquals("bb",rs.getString(3));
-            assertEquals(5,rs.getInt(4));
-            assertEquals("aa",rs.getString(5));
-            assertEquals(80,rs.getInt(6));
-            assertEquals(2,rs.getInt(7));   
-            assertTrue(rs.next());
-            assertEquals("a",rs.getString(1));  
-            assertEquals(40,rs.getInt(2));
-            assertEquals("aa",rs.getString(3));
-            assertEquals(10,rs.getInt(4));
-            assertEquals("bb",rs.getString(5));
-            assertEquals(20,rs.getInt(6));
-            assertEquals(1,rs.getInt(7));         
-            assertFalse(rs.next());  
-
-            query = "select * from " + tableName + " order by 7";
-            rs = conn.createStatement().executeQuery(query);
             assertTrue(rs.next());
-            assertEquals("a",rs.getString(1));  
-            assertEquals(40,rs.getInt(2));
-            assertEquals("aa",rs.getString(3));
-            assertEquals(10,rs.getInt(4));
-            assertEquals("bb",rs.getString(5));
-            assertEquals(20,rs.getInt(6));
-            assertEquals(1,rs.getInt(7));  
-            assertTrue(rs.next());
-            assertEquals("b",rs.getString(1));  
+            assertEquals("a",rs.getString(1));
             assertEquals(40,rs.getInt(2));
-            assertEquals("bb",rs.getString(3));
-            assertEquals(5,rs.getInt(4));
-            assertEquals("aa",rs.getString(5));
-            assertEquals(80,rs.getInt(6));
-            assertEquals(2,rs.getInt(7));  
-            assertTrue(rs.next());
-            assertEquals("c",rs.getString(1));
-            assertEquals(30,rs.getInt(2));
-            assertEquals("cc",rs.getString(3));
-            assertEquals(50,rs.getInt(4));
-            assertEquals("dd",rs.getString(5));
-            assertEquals(60,rs.getInt(6));
-            assertEquals(3,rs.getInt(7));
-            assertFalse(rs.next());  
+            assertFalse(rs.next());
         } finally {
             conn.close();
         }
@@ -343,19 +159,19 @@ public class OrderByIT extends ParallelStatsDisabledIT {
             String query = "select t1.* from " + tableName1 + " t1 join " + tableName2 + " t2 on t1.a_string = t2.a_string order by 3";
             ResultSet rs = conn.createStatement().executeQuery(query);
             assertTrue(rs.next());
-            assertEquals("a",rs.getString(1));  
+            assertEquals("a",rs.getString(1));
             assertEquals(40,rs.getInt(2));
             assertEquals("aa",rs.getString(3));
             assertEquals(10,rs.getInt(4));
             assertEquals("bb",rs.getString(5));
             assertEquals(20,rs.getInt(6));
             assertTrue(rs.next());
-            assertEquals("b",rs.getString(1));  
+            assertEquals("b",rs.getString(1));
             assertEquals(40,rs.getInt(2));
             assertEquals("bb",rs.getString(3));
             assertEquals(5,rs.getInt(4));
             assertEquals("aa",rs.getString(5));
-            assertEquals(80,rs.getInt(6));         
+            assertEquals(80,rs.getInt(6));
             assertTrue(rs.next());
             assertEquals("c",rs.getString(1));
             assertEquals(30,rs.getInt(2));
@@ -363,20 +179,20 @@ public class OrderByIT extends ParallelStatsDisabledIT {
             assertEquals(50,rs.getInt(4));
             assertEquals("dd",rs.getString(5));
             assertEquals(60,rs.getInt(6));
-            assertFalse(rs.next());  
+            assertFalse(rs.next());
 
             query = "select t1.a_string, t2.col1 from " + tableName1 + " t1 join " + tableName2 + " t2 on t1.a_string = t2.a_string order by 2";
             rs = conn.createStatement().executeQuery(query);
             assertTrue(rs.next());
-            assertEquals("b",rs.getString(1));  
+            assertEquals("b",rs.getString(1));
             assertEquals(20,rs.getInt(2));
             assertTrue(rs.next());
-            assertEquals("c",rs.getString(1));  
+            assertEquals("c",rs.getString(1));
             assertEquals(30,rs.getInt(2));
             assertTrue(rs.next());
-            assertEquals("a",rs.getString(1));  
+            assertEquals("a",rs.getString(1));
             assertEquals(40,rs.getInt(2));
-            assertFalse(rs.next()); 
+            assertFalse(rs.next());
         } finally {
             conn.close();
         }
@@ -441,24 +257,24 @@ public class OrderByIT extends ParallelStatsDisabledIT {
             String query = "select a_string, cf2.d from " + tableName1 + " union all select * from " + tableName2 + " order by 2";
             ResultSet rs = conn.createStatement().executeQuery(query);
             assertTrue(rs.next());
-            assertEquals("bb",rs.getString(1));  
+            assertEquals("bb",rs.getString(1));
             assertEquals(10,rs.getInt(2));
             assertTrue(rs.next());
-            assertEquals("a",rs.getString(1));  
-            assertEquals(20,rs.getInt(2));      
+            assertEquals("a",rs.getString(1));
+            assertEquals(20,rs.getInt(2));
             assertTrue(rs.next());
             assertEquals("cc",rs.getString(1));
             assertEquals(30,rs.getInt(2));
             assertTrue(rs.next());
-            assertEquals("aa",rs.getString(1));  
+            assertEquals("aa",rs.getString(1));
             assertEquals(40,rs.getInt(2));
             assertTrue(rs.next());
-            assertEquals("c",rs.getString(1));  
-            assertEquals(60,rs.getInt(2));      
+            assertEquals("c",rs.getString(1));
+            assertEquals(60,rs.getInt(2));
             assertTrue(rs.next());
             assertEquals("b",rs.getString(1));
             assertEquals(80,rs.getInt(2));
-            assertFalse(rs.next());  
+            assertFalse(rs.next());
         } finally {
             conn.close();
         }
@@ -514,7 +330,8 @@ public class OrderByIT extends ParallelStatsDisabledIT {
             conn.close();
         }
     }
-    
+
+
     @Test
     public void testOrderByRVC() throws Exception {
         Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
@@ -546,636 +363,102 @@ public class OrderByIT extends ParallelStatsDisabledIT {
     }
 
     @Test
-    public void testAggregateOrderBy() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        String tableName = generateUniqueName();
-        String ddl = "create table " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, VAL1 VARCHAR, VAL2 INTEGER)";
-        conn.createStatement().execute(ddl);
-
-        conn.createStatement().execute("upsert into " + tableName + " values ('ABC','aa123', 11)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('ABD','ba124', 1)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('ABE','cf125', 13)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('ABF','dan126', 4)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('ABG','elf127', 15)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('ABH','fan128', 6)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('AAA','get211', 100)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('AAB','hat212', 7)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('AAC','aap12', 2)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('AAD','ball12', 3)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('AAE','inn2110', 13)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('AAF','key2112', 40)");
-        conn.commit();
-
-        ResultSet rs;
-        PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class);
-        rs = stmt.executeQuery("select distinct ID, VAL1, VAL2 from " + tableName + " where ID in ('ABC','ABD','ABE','ABF','ABG','ABH','AAA', 'AAB', 'AAC','AAD','AAE','AAF') order by VAL1");
-        assertFalse(stmt.getQueryPlan().getOrderBy().getOrderByExpressions().isEmpty());
-        assertTrue(rs.next());
-        assertEquals("ABC", rs.getString(1));
-        assertEquals("aa123", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("aap12", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("ba124", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("ball12", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("cf125", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("dan126", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("elf127", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("fan128", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("get211", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("hat212", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("inn2110", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("AAF", rs.getString(1));
-        assertEquals("key2112", rs.getString(2));
-        assertFalse(rs.next());
-    }
-
-    @Test
-    public void testAggregateOptimizedOutOrderBy() throws Exception {
+    public void testColumnFamily() throws Exception {
         Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
         Connection conn = DriverManager.getConnection(getUrl(), props);
-        String tableName = generateUniqueName();
-        String ddl = "create table " + tableName + " (K1 VARCHAR NOT NULL, K2 VARCHAR NOT NULL, VAL1 VARCHAR, VAL2 INTEGER, CONSTRAINT pk PRIMARY KEY(K1,K2))";
-        conn.createStatement().execute(ddl);
-
-        conn.createStatement().execute("upsert into " + tableName + " values ('ABC','ABC','aa123', 11)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('ABD','ABC','ba124', 1)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('ABE','ABC','cf125', 13)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('ABF','ABC','dan126', 4)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('ABG','ABC','elf127', 15)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('ABH','ABC','fan128', 6)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('AAA','ABC','get211', 100)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('AAB','ABC','hat212', 7)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('AAC','ABC','aap12', 2)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('AAD','ABC','ball12', 3)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('AAE','ABC','inn2110', 13)");
-        conn.createStatement().execute("upsert into " + tableName + " values ('AAF','ABC','key2112', 40)");
-        conn.commit();
-
-        ResultSet rs;
-        PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class);
-        rs = stmt.executeQuery("select distinct K2, VAL1, VAL2 from " + tableName + " where K2 = 'ABC' order by VAL1");
-        assertTrue(stmt.getQueryPlan().getOrderBy().getOrderByExpressions().isEmpty());
-        assertTrue(rs.next());
-        assertEquals("ABC", rs.getString(1));
-        assertEquals("aa123", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("aap12", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("ba124", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("ball12", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("cf125", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("dan126", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("elf127", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("fan128", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("get211", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("hat212", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("inn2110", rs.getString(2));
-        assertTrue(rs.next());
-        assertEquals("ABC", rs.getString(1));
-        assertEquals("key2112", rs.getString(2));
-        assertFalse(rs.next());
-    }
-
-    @Test
-    public void testNullsLastWithDesc() throws Exception {
-        Connection conn=null;
-        try {
-            Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-            conn = DriverManager.getConnection(getUrl(), props);
-
-            String tableName=generateUniqueName();
-            String sql="CREATE TABLE "+tableName+" ( "+
-                "ORGANIZATION_ID VARCHAR,"+
-                "CONTAINER_ID VARCHAR,"+
-                "ENTITY_ID VARCHAR NOT NULL,"+
-                "CONSTRAINT TEST_PK PRIMARY KEY ( "+
-                "ORGANIZATION_ID DESC,"+
-                "CONTAINER_ID DESC,"+
-                "ENTITY_ID"+
-                "))";
-            conn.createStatement().execute(sql);
-
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('a',null,'11')");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'2','22')");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('c','3','33')");
-            conn.commit();
-
-            //-----ORGANIZATION_ID
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID ASC NULLS FIRST";
-            ResultSet rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{null,"a"},{"3","c"},});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null}});
-
-            //----CONTAINER_ID
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{"2",null},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"}});
-
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,'44')");
-            conn.commit();
-
-            //-----ORGANIZATION_ID ASC  CONTAINER_ID ASC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
-
-
-            //-----ORGANIZATION_ID ASC  CONTAINER_ID DESC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
-
-            //-----ORGANIZATION_ID DESC  CONTAINER_ID ASC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
-
-            //-----ORGANIZATION_ID DESC  CONTAINER_ID DESC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
-
-            //-----CONTAINER_ID ASC  ORGANIZATION_ID ASC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
-
-            //-----CONTAINER_ID ASC  ORGANIZATION_ID DESC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
-
-            //-----CONTAINER_ID DESC  ORGANIZATION_ID ASC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
-
-            //-----CONTAINER_ID DESC  ORGANIZATION_ID DESC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
-        } finally {
-            if(conn!=null) {
-                conn.close();
-            }
-        }
-    }
-
-    @Test
-    public void testOrderByReverseOptimization() throws Exception {
-        for(boolean salted: new boolean[]{true,false}) {
-            doTestOrderByReverseOptimization(salted,true,true,true);
-            doTestOrderByReverseOptimization(salted,true,true,false);
-            doTestOrderByReverseOptimization(salted,true,false,true);
-            doTestOrderByReverseOptimization(salted,true,false,false);
-            doTestOrderByReverseOptimization(salted,false,true,true);
-            doTestOrderByReverseOptimization(salted,false,true,false);
-            doTestOrderByReverseOptimization(salted,false,false,true);
-            doTestOrderByReverseOptimization(salted,false,false,false);
-        }
-    }
-
-    private void doTestOrderByReverseOptimization(boolean salted,boolean desc1,boolean desc2,boolean desc3) throws Exception {
-        Connection conn = null;
-        try {
-            Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-            conn = DriverManager.getConnection(getUrl(), props);
-            String tableName=generateUniqueName();
-            String sql="CREATE TABLE "+tableName+" ( "+
-                    "ORGANIZATION_ID INTEGER NOT NULL,"+
-                    "CONTAINER_ID INTEGER NOT NULL,"+
-                    "SCORE INTEGER NOT NULL,"+
-                    "ENTITY_ID INTEGER NOT NULL,"+
-                    "CONSTRAINT TEST_PK PRIMARY KEY ( "+
-                    "ORGANIZATION_ID" +(desc1 ? " DESC" : "" )+","+
-                    "CONTAINER_ID"+(desc2 ? " DESC" : "" )+","+
-                    "SCORE"+(desc3 ? " DESC" : "" )+","+
-                    "ENTITY_ID"+
-                    ")) "+(salted ? "SALT_BUCKETS =4" : "split on(4)");
-            conn.createStatement().execute(sql);
-
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,1,1,1)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,2,2,2)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,3,3,3)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (4,4,4,4)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (5,5,5,5)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (6,6,6,6)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,1,1,11)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,2,2,22)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,3,3,33)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (4,4,4,44)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (5,5,5,55)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (6,6,6,66)");
-            conn.commit();
-
-            //groupBy orderPreserving orderBy asc asc
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC, CONTAINER_ID ASC";
-            ResultSet rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
-
-            //groupBy orderPreserving orderBy asc desc
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC, CONTAINER_ID desc";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
-
-            //groupBy orderPreserving orderBy desc asc
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC, CONTAINER_ID DESC";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
-
-            //groupBy orderPreserving orderBy desc desc
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC, CONTAINER_ID DESC";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
-
-            //groupBy not orderPreserving orderBy asc asc
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC, SCORE ASC";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
-
-            //groupBy not orderPreserving orderBy asc desc
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC, SCORE ASC";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
-
-            //groupBy not orderPreserving orderBy desc asc
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC, SCORE DESC";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
-
-            //groupBy not orderPreserving orderBy desc desc
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC, SCORE DESC";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
-        } finally {
-            if(conn!=null) {
-                conn.close();
-            }
-        }
-    }
-
-    @Test
-    public void testOrderByReverseOptimizationWithNullsLast() throws Exception{
-        for(boolean salted: new boolean[]{true,false}) {
-            doTestOrderByReverseOptimizationWithNullsLast(salted,true,true,true);
-            doTestOrderByReverseOptimizationWithNullsLast(salted,true,true,false);
-            doTestOrderByReverseOptimizationWithNullsLast(salted,true,false,true);
-            doTestOrderByReverseOptimizationWithNullsLast(salted,true,false,false);
-            doTestOrderByReverseOptimizationWithNullsLast(salted,false,true,true);
-            doTestOrderByReverseOptimizationWithNullsLast(salted,false,true,false);
-            doTestOrderByReverseOptimizationWithNullsLast(salted,false,false,true);
-            doTestOrderByReverseOptimizationWithNullsLast(salted,false,false,false);
-        }
-    }
+        conn.setAutoCommit(false);
 
-    private void doTestOrderByReverseOptimizationWithNullsLast(boolean salted,boolean desc1,boolean desc2,boolean desc3) throws Exception {
-        Connection conn = null;
         try {
-            Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-            conn = DriverManager.getConnection(getUrl(), props);
-            String tableName=generateUniqueName();
-            String sql="CREATE TABLE "+tableName+" ( "+
-                    "ORGANIZATION_ID VARCHAR,"+
-                    "CONTAINER_ID VARCHAR,"+
-                    "SCORE VARCHAR,"+
-                    "ENTITY_ID VARCHAR NOT NULL,"+
-                    "CONSTRAINT TEST_PK PRIMARY KEY ( "+
-                    "ORGANIZATION_ID" +(desc1 ? " DESC" : "" )+","+
-                    "CONTAINER_ID"+(desc2 ? " DESC" : "" )+","+
-                    "SCORE"+(desc3 ? " DESC" : "" )+","+
-                    "ENTITY_ID"+
-                    ")) "+(salted ? "SALT_BUCKETS =4" : "split on('4')");
-            conn.createStatement().execute(sql);
-
-            for(int i=1;i<=6;i++) {
-                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'"+i+"','"+i+"','"+i+"')");
-                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'"+i+"',null,'"+i+"')");
-                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,'"+i+"','"+i+"')");
-                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,null,'"+i+"')");
-                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"','"+i+"','"+i+"','"+i+"')");
-                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"','"+i+"',null,'"+i+"')");
-                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"',null,'"+i+"','"+i+"')");
-                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"',null,null,'"+i+"')");
-            }
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,null,'66')");
+            String tableName = generateUniqueName();
+            String ddl = "CREATE TABLE " + tableName +
+                    "  (a_string varchar not null, cf1.a integer, cf1.b varchar, col1 integer, cf2.c varchar, cf2.d integer, col2 integer" +
+                    "  CONSTRAINT pk PRIMARY KEY (a_string))\n";
+            createTestTable(getUrl(), ddl);
+            String dml = "UPSERT INTO " + tableName + " VALUES(?,?,?,?,?,?,?)";
+            PreparedStatement stmt = conn.prepareStatement(dml);
+            stmt.setString(1, "a");
+            stmt.setInt(2, 40);
+            stmt.setString(3, "aa");
+            stmt.setInt(4, 10);
+            stmt.setString(5, "bb");
+            stmt.setInt(6, 20);
+            stmt.setInt(7, 1);
+            stmt.execute();
+            stmt.setString(1, "c");
+            stmt.setInt(2, 30);
+            stmt.setString(3, "cc");
+            stmt.setInt(4, 50);
+            stmt.setString(5, "dd");
+            stmt.setInt(6, 60);
+            stmt.setInt(7, 3);
+            stmt.execute();
+            stmt.setString(1, "b");
+            stmt.setInt(2, 40);
+            stmt.setString(3, "bb");
+            stmt.setInt(4, 5);
+            stmt.setString(5, "aa");
+            stmt.setInt(6, 80);
+            stmt.setInt(7, 2);
+            stmt.execute();
             conn.commit();
 
-            //groupBy orderPreserving orderBy asc asc
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST";
-            ResultSet rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID ASC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"},{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"}});
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
-
-            //groupBy orderPreserving orderBy asc desc
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
-
-            //groupBy orderPreserving orderBy desc asc
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"}});
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
-
-            //groupBy orderPreserving orderBy desc desc
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
-
-            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
-
-            //-----groupBy not orderPreserving--
-
-            //groupBy not orderPreserving orderBy asc asc
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"},{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"}});
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
-
-            //groupBy not orderPreserving orderBy asc desc
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
-
-            //groupBy not orderPreserving orderBy desc asc
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"}});
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
-
-            //groupBy not orderPreserving orderBy desc desc
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
-
-            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
-
-            //-------test only one return column----------------------------------
-
-            sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE ASC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null},{"1"},{"2"},{"3"},{"4"},{"5"},{"6"}});
-
-            sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"1"},{"2"},{"3"},{"4"},{"5"},{"6"},{null}});
-
-            sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{null},{"6"},{"5"},{"4"},{"3"},{"2"},{"1"}});
+            String query = "select * from " + tableName + " order by 2, 5";
+            ResultSet rs = conn.createStatement().executeQuery(query);
+            assertTrue(rs.next());
+            assertEquals("c",rs.getString(1));
+            assertEquals(30,rs.getInt(2));
+            assertEquals("cc",rs.getString(3));
+            assertEquals(50,rs.getInt(4));
+            assertEquals("dd",rs.getString(5));
+            assertEquals(60,rs.getInt(6));
+            assertEquals(3,rs.getInt(7));
+            assertTrue(rs.next());
+            assertEquals("b",rs.getString(1));
+            assertEquals(40,rs.getInt(2));
+            assertEquals("bb",rs.getString(3));
+            assertEquals(5,rs.getInt(4));
+            assertEquals("aa",rs.getString(5));
+            assertEquals(80,rs.getInt(6));
+            assertEquals(2,rs.getInt(7));
+            assertTrue(rs.next());
+            assertEquals("a",rs.getString(1));
+            assertEquals(40,rs.getInt(2));
+            assertEquals("aa",rs.getString(3));
+            assertEquals(10,rs.getInt(4));
+            assertEquals("bb",rs.getString(5));
+            assertEquals(20,rs.getInt(6));
+            assertEquals(1,rs.getInt(7));
+            assertFalse(rs.next());
 
-            sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{"6"},{"5"},{"4"},{"3"},{"2"},{"1"},{null}});
+            query = "select * from " + tableName + " order by 7";
+            rs = conn.createStatement().executeQuery(query);
+            assertTrue(rs.next());
+            assertEquals("a",rs.getString(1));
+            assertEquals(40,rs.getInt(2));
+            assertEquals("aa",rs.getString(3));
+            assertEquals(10,rs.getInt(4));
+            assertEquals("bb",rs.getString(5));
+            assertEquals(20,rs.getInt(6));
+            assertEquals(1,rs.getInt(7));
+            assertTrue(rs.next());
+            assertEquals("b",rs.getString(1));
+            assertEquals(40,rs.getInt(2));
+            assertEquals("bb",rs.getString(3));
+            assertEquals(5,rs.getInt(4));
+            assertEquals("aa",rs.getString(5));
+            assertEquals(80,rs.getInt(6));
+            assertEquals(2,rs.getInt(7));
+            assertTrue(rs.next());
+            assertEquals("c",rs.getString(1));
+            assertEquals(30,rs.getInt(2));
+            assertEquals("cc",rs.getString(3));
+            assertEquals(50,rs.getInt(4));
+            assertEquals("dd",rs.getString(5));
+            assertEquals(60,rs.getInt(6));
+            assertEquals(3,rs.getInt(7));
+            assertFalse(rs.next());
         } finally {
-            if(conn!=null) {
-                conn.close();
-            }
+            conn.close();
         }
     }
-
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledIT.java
index 98939da..561aee5 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledIT.java
@@ -18,12 +18,26 @@
 
 package org.apache.phoenix.end2end;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.util.QueryBuilder;
+import org.apache.phoenix.util.QueryUtil;
 import org.apache.phoenix.util.ReadOnlyProps;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.experimental.categories.Category;
 
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+
+
 /**
  * Base class for tests whose methods run in parallel with statistics disabled.
  * You must create unique names using {@link #generateUniqueName()} for each
@@ -41,4 +55,30 @@ public abstract class ParallelStatsDisabledIT extends BaseTest {
     public static void tearDownMiniCluster() throws Exception {
         BaseTest.tearDownMiniClusterIfBeyondThreshold();
     }
+
+    protected ResultSet executeQuery(Connection conn, QueryBuilder queryBuilder) throws SQLException {
+        PreparedStatement statement = conn.prepareStatement(queryBuilder.build());
+        ResultSet rs = statement.executeQuery();
+        return rs;
+    }
+
+    protected ResultSet executeQueryThrowsException(Connection conn, QueryBuilder queryBuilder,
+            String expectedPhoenixExceptionMsg, String expectedSparkExceptionMsg) {
+        ResultSet rs = null;
+        try {
+            rs = executeQuery(conn, queryBuilder);
+            fail();
+        }
+        catch(Exception e) {
+            assertTrue(e.getMessage().contains(expectedPhoenixExceptionMsg));
+        }
+        return rs;
+    }
+
+    protected void validateQueryPlan(Connection conn, QueryBuilder queryBuilder, String expectedPhoenixPlan, String expectedSparkPlan) throws SQLException {
+        if (StringUtils.isNotBlank(expectedPhoenixPlan)) {
+            ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + queryBuilder.build());
+            assertEquals(expectedPhoenixPlan, QueryUtil.getExplainPlan(rs));
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/BaseSaltedTableIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/BaseSaltedTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/BaseSaltedTableIT.java
new file mode 100644
index 0000000..3051cd6
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/BaseSaltedTableIT.java
@@ -0,0 +1,474 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end.salted;
+
+import static org.apache.phoenix.util.TestUtil.TABLE_WITH_SALTING;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.Properties;
+
+import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryBuilder;
+import org.apache.phoenix.util.QueryUtil;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public abstract class BaseSaltedTableIT extends ParallelStatsDisabledIT  {
+
+    protected static String initTableValues(byte[][] splits) throws Exception {
+        String tableName = generateUniqueName();
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+
+        // Rows we inserted:
+        // 1ab123abc111
+        // 1abc456abc111
+        // 1de123abc111
+        // 2abc123def222
+        // 3abc123ghi333
+        // 4abc123jkl444
+        try {
+            // Upsert with no column specifies.
+            ensureTableCreated(getUrl(), tableName, TABLE_WITH_SALTING, splits, null, null);
+            String query = "UPSERT INTO " + tableName + " VALUES(?,?,?,?,?)";
+            PreparedStatement stmt = conn.prepareStatement(query);
+            stmt.setInt(1, 1);
+            stmt.setString(2, "ab");
+            stmt.setString(3, "123");
+            stmt.setString(4, "abc");
+            stmt.setInt(5, 111);
+            stmt.execute();
+            conn.commit();
+
+            stmt.setInt(1, 1);
+            stmt.setString(2, "abc");
+            stmt.setString(3, "456");
+            stmt.setString(4, "abc");
+            stmt.setInt(5, 111);
+            stmt.execute();
+            conn.commit();
+
+            // Test upsert when statement explicitly specifies the columns to upsert into.
+            query = "UPSERT INTO " + tableName +
+                    " (a_integer, a_string, a_id, b_string, b_integer) " +
+                    " VALUES(?,?,?,?,?)";
+            stmt = conn.prepareStatement(query);
+
+            stmt.setInt(1, 1);
+            stmt.setString(2, "de");
+            stmt.setString(3, "123");
+            stmt.setString(4, "abc");
+            stmt.setInt(5, 111);
+            stmt.execute();
+            conn.commit();
+
+            stmt.setInt(1, 2);
+            stmt.setString(2, "abc");
+            stmt.setString(3, "123");
+            stmt.setString(4, "def");
+            stmt.setInt(5, 222);
+            stmt.execute();
+            conn.commit();
+
+            // Test upsert when order of column is shuffled.
+            query = "UPSERT INTO " + tableName +
+                    " (a_string, a_integer, a_id, b_string, b_integer) " +
+                    " VALUES(?,?,?,?,?)";
+            stmt = conn.prepareStatement(query);
+            stmt.setString(1, "abc");
+            stmt.setInt(2, 3);
+            stmt.setString(3, "123");
+            stmt.setString(4, "ghi");
+            stmt.setInt(5, 333);
+            stmt.execute();
+            conn.commit();
+
+            stmt.setString(1, "abc");
+            stmt.setInt(2, 4);
+            stmt.setString(3, "123");
+            stmt.setString(4, "jkl");
+            stmt.setInt(5, 444);
+            stmt.execute();
+            conn.commit();
+        } finally {
+            conn.close();
+        }
+        return tableName;
+    }
+
+    @Test
+    public void testSelectValueNoWhereClause() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        try {
+            String tableName = initTableValues(null);
+            // "SELECT * FROM " + tableName;
+            QueryBuilder queryBuilder = new QueryBuilder()
+                .setSelectColumns(
+                    Lists.newArrayList("A_INTEGER", "A_STRING", "A_ID", "B_STRING", "B_INTEGER"))
+                .setFullTableName(tableName);
+            ResultSet rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertEquals(1, rs.getInt(1));
+            assertEquals("ab", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("abc", rs.getString(4));
+            assertEquals(111, rs.getInt(5));
+
+            assertTrue(rs.next());
+            assertEquals(1, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+            assertEquals("456", rs.getString(3));
+            assertEquals("abc", rs.getString(4));
+            assertEquals(111, rs.getInt(5));
+
+            assertTrue(rs.next());
+            assertEquals(1, rs.getInt(1));
+            assertEquals("de", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("abc", rs.getString(4));
+            assertEquals(111, rs.getInt(5));
+
+            assertTrue(rs.next());
+            assertEquals(2, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("def", rs.getString(4));
+            assertEquals(222, rs.getInt(5));
+
+            assertTrue(rs.next());
+            assertEquals(3, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("ghi", rs.getString(4));
+            assertEquals(333, rs.getInt(5));
+
+            assertTrue(rs.next());
+            assertEquals(4, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("jkl", rs.getString(4));
+            assertEquals(444, rs.getInt(5));
+
+            assertFalse(rs.next());
+        } finally {
+            conn.close();
+        }
+    }
+
+    @Test
+    public void testSelectValueWithFullyQualifiedWhereClause() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        try {
+            String tableName = initTableValues(null);
+            PreparedStatement stmt;
+            ResultSet rs;
+
+            // Variable length slot with bounded ranges.
+            QueryBuilder queryBuilder = new QueryBuilder()
+                .setSelectColumns(
+                        Lists.newArrayList("A_INTEGER", "A_STRING", "A_ID", "B_STRING", "B_INTEGER"))
+                .setFullTableName(tableName)
+                .setWhereClause("a_integer = 1 AND a_string >= 'ab' AND a_string < 'de' AND a_id = '123'");
+            rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertEquals(1, rs.getInt(1));
+            assertEquals("ab", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("abc", rs.getString(4));
+            assertEquals(111, rs.getInt(5));
+            assertFalse(rs.next());
+
+            // all single slots with one value.
+            queryBuilder.setWhereClause("a_integer = 1 AND a_string = 'ab' AND a_id = '123'");
+            rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertEquals(1, rs.getInt(1));
+            assertEquals("ab", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("abc", rs.getString(4));
+            assertEquals(111, rs.getInt(5));
+            assertFalse(rs.next());
+
+            // all single slots with multiple values.
+            queryBuilder.setWhereClause("a_integer in (2, 4) AND a_string = 'abc' AND a_id = '123'");
+            rs = executeQuery(conn, queryBuilder);
+
+            assertTrue(rs.next());
+            assertEquals(2, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("def", rs.getString(4));
+            assertEquals(222, rs.getInt(5));
+
+            assertTrue(rs.next());
+            assertEquals(4, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("jkl", rs.getString(4));
+            assertEquals(444, rs.getInt(5));
+            assertFalse(rs.next());
+
+            queryBuilder.setWhereClause("A_INTEGER in (1,2,3,4) AND A_STRING in ('a', 'abc', 'de') AND A_ID = '123'");
+            queryBuilder.setSelectColumns(Lists.newArrayList("A_INTEGER", "A_STRING", "A_ID"));
+            rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertEquals(1, rs.getInt(1));
+            assertEquals("de", rs.getString(2));
+
+            assertTrue(rs.next());
+            assertEquals(2, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+
+            assertTrue(rs.next());
+            assertEquals(3, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+
+            assertTrue(rs.next());
+            assertEquals(4, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+            assertFalse(rs.next());
+
+            // fixed length slot with bounded ranges.
+            queryBuilder.setWhereClause("A_INTEGER > 1 AND A_INTEGER < 4 AND A_STRING = 'abc' AND A_ID = '123'");
+            queryBuilder.setSelectColumns(Lists.newArrayList("A_STRING", "A_ID", "A_INTEGER"));
+            rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertEquals("abc", rs.getString(1));
+            assertEquals("123", rs.getString(2));
+
+            assertTrue(rs.next());
+            assertEquals("abc", rs.getString(1));
+            assertEquals("123", rs.getString(2));
+            assertFalse(rs.next());
+
+            // fixed length slot with unbound ranges.
+            queryBuilder.setWhereClause("A_INTEGER > 1 AND A_STRING = 'abc' AND A_ID = '123'");
+            queryBuilder.setSelectColumns(Lists.newArrayList("B_STRING", "B_INTEGER", "A_INTEGER", "A_STRING", "A_ID"));
+            rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertEquals("def", rs.getString(1));
+            assertEquals(222, rs.getInt(2));
+
+            assertTrue(rs.next());
+            assertEquals("ghi", rs.getString(1));
+            assertEquals(333, rs.getInt(2));
+
+            assertTrue(rs.next());
+            assertEquals("jkl", rs.getString(1));
+            assertEquals(444, rs.getInt(2));
+            assertFalse(rs.next());
+
+            // Variable length slot with unbounded ranges.
+            queryBuilder.setWhereClause("A_INTEGER = 1 AND A_STRING > 'ab' AND A_ID = '123'");
+            queryBuilder.setSelectColumns(
+                Lists.newArrayList("A_INTEGER", "A_STRING", "A_ID", "B_STRING", "B_INTEGER"));
+            rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertEquals(1, rs.getInt(1));
+            assertEquals("de", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("abc", rs.getString(4));
+            assertEquals(111, rs.getInt(5));
+            assertFalse(rs.next());
+
+        } finally {
+            conn.close();
+        }
+    }
+
+    @Test
+    public void testSelectValueWithNotFullyQualifiedWhereClause() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        try {
+            String tableName = initTableValues(null);
+
+            // Where without fully qualified key, point query.
+            String query = "SELECT * FROM " + tableName + " WHERE a_integer = ? AND a_string = ?";
+            PreparedStatement stmt = conn.prepareStatement(query);
+
+            stmt.setInt(1, 1);
+            stmt.setString(2, "abc");
+            ResultSet rs = stmt.executeQuery();
+            assertTrue(rs.next());
+            assertEquals(1, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+            assertEquals("456", rs.getString(3));
+            assertEquals("abc", rs.getString(4));
+            assertEquals(111, rs.getInt(5));
+            assertFalse(rs.next());
+
+            // Where without fully qualified key, range query.
+            query = "SELECT * FROM " + tableName + " WHERE a_integer >= 2";
+            stmt = conn.prepareStatement(query);
+            rs = stmt.executeQuery();
+            assertTrue(rs.next());
+            assertEquals(2, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("def", rs.getString(4));
+            assertEquals(222, rs.getInt(5));
+
+            assertTrue(rs.next());
+            assertEquals(3, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("ghi", rs.getString(4));
+            assertEquals(333, rs.getInt(5));
+
+            assertTrue(rs.next());
+            assertEquals(4, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("jkl", rs.getString(4));
+            assertEquals(444, rs.getInt(5));
+            assertFalse(rs.next());
+
+            // With point query.
+            query = "SELECT a_string FROM " + tableName + " WHERE a_string = ?";
+            stmt = conn.prepareStatement(query);
+            stmt.setString(1, "de");
+            rs = stmt.executeQuery();
+            assertTrue(rs.next());
+            assertEquals("de", rs.getString(1));
+            assertFalse(rs.next());
+
+            query = "SELECT a_id FROM " + tableName + " WHERE a_id = ?";
+            stmt = conn.prepareStatement(query);
+            stmt.setString(1, "456");
+            rs = stmt.executeQuery();
+            assertTrue(rs.next());
+            assertEquals("456", rs.getString(1));
+            assertFalse(rs.next());
+        } finally {
+            conn.close();
+        }
+    }
+
+    @Test
+    public void testSelectWithGroupBy() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        try {
+            String tableName = initTableValues(null);
+
+            String query = "SELECT a_integer FROM " + tableName + " GROUP BY a_integer";
+            PreparedStatement stmt = conn.prepareStatement(query);
+            ResultSet rs = stmt.executeQuery();
+            int count = 0;
+            while (rs.next()) {
+                count++;
+            }
+            assertEquals("Group by does not return the right count.", count, 4);
+        } finally {
+            conn.close();
+        }
+    }
+
+    @Test
+    public void testLimitScan() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        try {
+            String tableName = initTableValues(null);
+
+            String query = "SELECT a_integer FROM " + tableName + " WHERE a_string='abc' LIMIT 1";
+            PreparedStatement stmt = conn.prepareStatement(query);
+            ResultSet rs = stmt.executeQuery();
+            assertTrue(rs.next());
+            assertEquals(1, rs.getInt(1));
+            assertFalse(rs.next());
+        } finally {
+            conn.close();
+        }
+    }
+
+    @Test
+    public void testSelectWithOrderByRowKey() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        try {
+            String tableName = initTableValues(null);
+
+            String query = "SELECT * FROM " + tableName + " ORDER  BY  a_integer, a_string, a_id";
+            PreparedStatement statement = conn.prepareStatement(query);
+            ResultSet explainPlan = statement.executeQuery("EXPLAIN " + query);
+            // Confirm that ORDER BY in row key order will be optimized out for salted table
+            assertEquals("CLIENT PARALLEL 4-WAY FULL SCAN OVER " + tableName + "\n" +
+                    "CLIENT MERGE SORT", QueryUtil.getExplainPlan(explainPlan));
+            ResultSet rs = statement.executeQuery();
+
+            assertTrue(rs.next());
+            assertEquals(1, rs.getInt(1));
+            assertEquals("ab", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("abc", rs.getString(4));
+            assertEquals(111, rs.getInt(5));
+
+            assertTrue(rs.next());
+            assertEquals(1, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+            assertEquals("456", rs.getString(3));
+            assertEquals("abc", rs.getString(4));
+            assertEquals(111, rs.getInt(5));
+
+            assertTrue(rs.next());
+            assertEquals(1, rs.getInt(1));
+            assertEquals("de", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("abc", rs.getString(4));
+            assertEquals(111, rs.getInt(5));
+
+            assertTrue(rs.next());
+            assertEquals(2, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("def", rs.getString(4));
+            assertEquals(222, rs.getInt(5));
+
+            assertTrue(rs.next());
+            assertEquals(3, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("ghi", rs.getString(4));
+            assertEquals(333, rs.getInt(5));
+
+            assertTrue(rs.next());
+            assertEquals(4, rs.getInt(1));
+            assertEquals("abc", rs.getString(2));
+            assertEquals("123", rs.getString(3));
+            assertEquals("jkl", rs.getString(4));
+            assertEquals(444, rs.getInt(5));
+
+            assertFalse(rs.next());
+        } finally {
+            conn.close();
+        }
+    }
+}


[07/22] phoenix git commit: PHOENIX-4909 Missing the phoenix-loadbalancer module dependency in phoenix-assembly/pom.xml

Posted by ja...@apache.org.
PHOENIX-4909 Missing the phoenix-loadbalancer module dependency in phoenix-assembly/pom.xml

Closes #352

Signed-off-by: Josh Elser <el...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/fc35ccec
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/fc35ccec
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/fc35ccec

Branch: refs/heads/omid2
Commit: fc35ccecc6fa497da506aff48d601657e41d2a70
Parents: c936d81
Author: Vitaliy <vi...@mail.ru>
Authored: Wed Sep 19 01:09:01 2018 +0300
Committer: Josh Elser <el...@apache.org>
Committed: Wed Oct 31 12:20:52 2018 -0400

----------------------------------------------------------------------
 bin/phoenix_utils.py                                      | 2 ++
 phoenix-assembly/pom.xml                                  | 4 ++++
 phoenix-assembly/src/build/components/all-common-jars.xml | 8 ++++++++
 3 files changed, 14 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc35ccec/bin/phoenix_utils.py
----------------------------------------------------------------------
diff --git a/bin/phoenix_utils.py b/bin/phoenix_utils.py
index aa04a5b..98a0896 100755
--- a/bin/phoenix_utils.py
+++ b/bin/phoenix_utils.py
@@ -166,6 +166,8 @@ def setPath():
     global phoenix_loadbalancer_jar
     phoenix_loadbalancer_jar = find(PHOENIX_LOADBALANCER_JAR_PATTERN, os.path.join(current_dir, "..", "phoenix-loadbalancer", "target", "*"))
     if phoenix_loadbalancer_jar == "":
+        phoenix_loadbalancer_jar = findFileInPathWithoutRecursion(PHOENIX_LOADBALANCER_JAR_PATTERN, os.path.join(current_dir, "..", "lib"))
+    if phoenix_loadbalancer_jar == "":
         phoenix_loadbalancer_jar = findFileInPathWithoutRecursion(PHOENIX_LOADBALANCER_JAR_PATTERN, os.path.join(current_dir, ".."))
 
     global phoenix_traceserver_jar

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc35ccec/phoenix-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/pom.xml b/phoenix-assembly/pom.xml
index f11645a..e41d7e6 100644
--- a/phoenix-assembly/pom.xml
+++ b/phoenix-assembly/pom.xml
@@ -126,5 +126,9 @@
       <groupId>org.apache.phoenix</groupId>
       <artifactId>phoenix-queryserver-client</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.phoenix</groupId>
+      <artifactId>phoenix-load-balancer</artifactId>
+    </dependency>
   </dependencies>
 </project>

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc35ccec/phoenix-assembly/src/build/components/all-common-jars.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/components/all-common-jars.xml b/phoenix-assembly/src/build/components/all-common-jars.xml
index 3d27b26..08ca29a 100644
--- a/phoenix-assembly/src/build/components/all-common-jars.xml
+++ b/phoenix-assembly/src/build/components/all-common-jars.xml
@@ -158,5 +158,13 @@
       </includes>
       <fileMode>0644</fileMode>
     </fileSet>
+    <fileSet>
+      <directory>${project.basedir}/../phoenix-load-balancer/target/</directory>
+      <outputDirectory>lib</outputDirectory>
+      <includes>
+        <include>phoenix-*.jar</include>
+      </includes>
+      <fileMode>0644</fileMode>
+    </fileSet>
   </fileSets>
 </component>


[15/22] phoenix git commit: PHOENIX-4981 Add tests for ORDER BY, GROUP BY and salted tables using phoenix-spark

Posted by ja...@apache.org.
PHOENIX-4981 Add tests for ORDER BY, GROUP BY and salted tables using phoenix-spark


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/9bfaf183
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/9bfaf183
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/9bfaf183

Branch: refs/heads/omid2
Commit: 9bfaf183a3d092bce6b2dfcacd77ba46aa1b078b
Parents: 1b2a3d5
Author: Thomas D'Silva <td...@apache.org>
Authored: Thu Oct 18 22:00:01 2018 -0700
Committer: Thomas D'Silva <td...@apache.org>
Committed: Tue Nov 6 14:51:13 2018 -0800

----------------------------------------------------------------------
 .../org/apache/phoenix/end2end/AggregateIT.java |  987 +---------------
 .../apache/phoenix/end2end/BaseAggregateIT.java | 1022 +++++++++++++++++
 .../apache/phoenix/end2end/BaseOrderByIT.java   |  940 ++++++++++++++++
 .../org/apache/phoenix/end2end/OrderByIT.java   |  943 ++--------------
 .../end2end/ParallelStatsDisabledIT.java        |   40 +
 .../end2end/salted/BaseSaltedTableIT.java       |  474 ++++++++
 .../phoenix/end2end/salted/SaltedTableIT.java   |  450 +-------
 .../org/apache/phoenix/util/QueryBuilder.java   |  211 ++++
 .../java/org/apache/phoenix/util/QueryUtil.java |   38 +-
 .../index/IndexScrutinyTableOutputTest.java     |    6 +-
 .../util/PhoenixConfigurationUtilTest.java      |    6 +-
 .../org/apache/phoenix/util/QueryUtilTest.java  |   10 +-
 phoenix-spark/pom.xml                           |    8 +
 .../org/apache/phoenix/spark/AggregateIT.java   |   91 ++
 .../org/apache/phoenix/spark/OrderByIT.java     |  460 ++++++++
 .../org/apache/phoenix/spark/SaltedTableIT.java |   53 +
 .../org/apache/phoenix/spark/SparkUtil.java     |   87 ++
 .../apache/phoenix/spark/PhoenixSparkIT.scala   |    9 +-
 .../apache/phoenix/spark/SparkResultSet.java    | 1056 ++++++++++++++++++
 .../org/apache/phoenix/spark/PhoenixRDD.scala   |   27 +-
 pom.xml                                         |    2 +-
 21 files changed, 4650 insertions(+), 2270 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateIT.java
index 2059311..8916d4d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateIT.java
@@ -18,506 +18,28 @@
 package org.apache.phoenix.end2end;
 
 import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.apache.phoenix.util.TestUtil.assertResultSet;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static org.apache.phoenix.util.TestUtil.assertResultSet;
 
-import java.io.IOException;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.List;
 import java.util.Properties;
 
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.compile.QueryPlan;
-import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
-import org.apache.phoenix.jdbc.PhoenixStatement;
-import org.apache.phoenix.query.KeyRange;
 import org.apache.phoenix.schema.AmbiguousColumnException;
-import org.apache.phoenix.schema.types.PChar;
-import org.apache.phoenix.schema.types.PInteger;
-import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.PropertiesUtil;
-import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.QueryBuilder;
 import org.apache.phoenix.util.TestUtil;
 import org.junit.Test;
 
+public class AggregateIT extends BaseAggregateIT {
 
-public class AggregateIT extends ParallelStatsDisabledIT {
-    private static void initData(Connection conn, String tableName) throws SQLException {
-        conn.createStatement().execute("create table " + tableName +
-                "   (id varchar not null primary key,\n" +
-                "    uri varchar, appcpu integer)");
-        insertRow(conn, tableName, "Report1", 10, 1);
-        insertRow(conn, tableName, "Report2", 10, 2);
-        insertRow(conn, tableName, "Report3", 30, 3);
-        insertRow(conn, tableName, "Report4", 30, 4);
-        insertRow(conn, tableName, "SOQL1", 10, 5);
-        insertRow(conn, tableName, "SOQL2", 10, 6);
-        insertRow(conn, tableName, "SOQL3", 30, 7);
-        insertRow(conn, tableName, "SOQL4", 30, 8);
-        conn.commit();
-    }
-
-    private static void insertRow(Connection conn, String tableName, String uri, int appcpu, int id) throws SQLException {
-        PreparedStatement statement = conn.prepareStatement("UPSERT INTO " + tableName + "(id, uri, appcpu) values (?,?,?)");
-        statement.setString(1, "id" + id);
-        statement.setString(2, uri);
-        statement.setInt(3, appcpu);
-        statement.executeUpdate();
-    }
-
-    @Test
-    public void testDuplicateTrailingAggExpr() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        String tableName = generateUniqueName();
-        conn.createStatement().execute("create table " + tableName +
-                "   (nam VARCHAR(20), address VARCHAR(20), id BIGINT "
-                + "constraint my_pk primary key (id))");
-        PreparedStatement statement = conn.prepareStatement("UPSERT INTO " + tableName + "(nam, address, id) values (?,?,?)");
-        statement.setString(1, "pulkit");
-        statement.setString(2, "badaun");
-        statement.setInt(3, 1);
-        statement.executeUpdate();
-        conn.commit();
-        Statement stmt = conn.createStatement();
-        ResultSet rs = stmt.executeQuery("select distinct 'harshit' as \"test_column\", trim(nam), trim(nam) from " + tableName);
-        assertTrue(rs.next());
-        assertEquals("harshit", rs.getString(1));
-        assertEquals("pulkit", rs.getString(2));
-        assertEquals("pulkit", rs.getString(3));
-        conn.close();
-    }
-
-    @Test
-    public void testExpressionInGroupBy() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        String tableName = generateUniqueName();
-        String ddl = " create table " + tableName + "(tgb_id integer NOT NULL,utc_date_epoch integer NOT NULL,tgb_name varchar(40),ack_success_count integer" +
-                ",ack_success_one_ack_count integer, CONSTRAINT pk_tgb_counter PRIMARY KEY(tgb_id, utc_date_epoch))";
-        String query = "SELECT tgb_id, tgb_name, (utc_date_epoch/10)*10 AS utc_epoch_hour,SUM(ack_success_count + ack_success_one_ack_count) AS ack_tx_sum" +
-                " FROM " + tableName + " GROUP BY tgb_id, tgb_name, utc_epoch_hour";
-
-        createTestTable(getUrl(), ddl);
-        String dml = "UPSERT INTO " + tableName + " VALUES(?,?,?,?,?)";
-        PreparedStatement stmt = conn.prepareStatement(dml);
-        stmt.setInt(1, 1);
-        stmt.setInt(2, 1000);
-        stmt.setString(3, "aaa");
-        stmt.setInt(4, 1);
-        stmt.setInt(5, 1);
-        stmt.execute();
-        stmt.setInt(1, 2);
-        stmt.setInt(2, 2000);
-        stmt.setString(3, "bbb");
-        stmt.setInt(4, 2);
-        stmt.setInt(5, 2);
-        stmt.execute();
-        conn.commit();
-
-        ResultSet rs = conn.createStatement().executeQuery(query);
-        assertTrue(rs.next());
-        assertEquals(1,rs.getInt(1));
-        assertEquals("aaa",rs.getString(2));
-        assertEquals(1000,rs.getInt(3));
-        assertEquals(2,rs.getInt(4));
-        assertTrue(rs.next());
-        assertEquals(2,rs.getInt(1));
-        assertEquals("bbb",rs.getString(2));
-        assertEquals(2000,rs.getInt(3));
-        assertEquals(4,rs.getInt(4));
-        assertFalse(rs.next());
-        rs.close();
-        conn.close();
-    }
-    
-    @Test
-    public void testBooleanInGroupBy() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        String tableName = generateUniqueName();
-        String ddl = " create table " + tableName + "(id varchar primary key,v1 boolean, v2 integer, v3 integer)";
-
-        createTestTable(getUrl(), ddl);
-        PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName + "(id,v2,v3) VALUES(?,?,?)");
-        stmt.setString(1, "a");
-        stmt.setInt(2, 1);
-        stmt.setInt(3, 1);
-        stmt.execute();
-        stmt.close();
-        stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES(?,?,?,?)");
-        stmt.setString(1, "b");
-        stmt.setBoolean(2, false);
-        stmt.setInt(3, 2);
-        stmt.setInt(4, 2);
-        stmt.execute();
-        stmt.setString(1, "c");
-        stmt.setBoolean(2, true);
-        stmt.setInt(3, 3);
-        stmt.setInt(4, 3);
-        stmt.execute();
-        conn.commit();
-
-        String[] gbs = {"v1,v2,v3","v1,v3,v2","v2,v1,v3"};
-        for (String gb : gbs) {
-            ResultSet rs = conn.createStatement().executeQuery("SELECT v1, v2, v3 from " + tableName + " group by " + gb);
-            assertTrue(rs.next());
-            assertEquals(false,rs.getBoolean("v1"));
-            assertTrue(rs.wasNull());
-            assertEquals(1,rs.getInt("v2"));
-            assertEquals(1,rs.getInt("v3"));
-            assertTrue(rs.next());
-            assertEquals(false,rs.getBoolean("v1"));
-            assertFalse(rs.wasNull());
-            assertEquals(2,rs.getInt("v2"));
-            assertEquals(2,rs.getInt("v3"));
-            assertTrue(rs.next());
-            assertEquals(true,rs.getBoolean("v1"));
-            assertEquals(3,rs.getInt("v2"));
-            assertEquals(3,rs.getInt("v3"));
-            assertFalse(rs.next());
-            rs.close();
-        }
-        conn.close();
-    }
-    
-    @Test
-    public void testScanUri() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        String tableName = generateUniqueName();
-        initData(conn, tableName);
-        Statement stmt = conn.createStatement();
-        ResultSet rs = stmt.executeQuery("select uri from " + tableName);
-        assertTrue(rs.next());
-        assertEquals("Report1", rs.getString(1));
-        assertTrue(rs.next());
-        assertEquals("Report2", rs.getString(1));
-        assertTrue(rs.next());
-        assertEquals("Report3", rs.getString(1));
-        assertTrue(rs.next());
-        assertEquals("Report4", rs.getString(1));
-        assertTrue(rs.next());
-        assertEquals("SOQL1", rs.getString(1));
-        assertTrue(rs.next());
-        assertEquals("SOQL2", rs.getString(1));
-        assertTrue(rs.next());
-        assertEquals("SOQL3", rs.getString(1));
-        assertTrue(rs.next());
-        assertEquals("SOQL4", rs.getString(1));
-        assertFalse(rs.next());
-        conn.close();
-    }
-
-    @Test
-    public void testCount() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        String tableName = generateUniqueName();
-        initData(conn, tableName);
-        Statement stmt = conn.createStatement();
-        ResultSet rs = stmt.executeQuery("select count(1) from " + tableName);
-        assertTrue(rs.next());
-        assertEquals(8, rs.getInt(1));
-        assertFalse(rs.next());
-        conn.close();
-    }
-
-    @Test
-    public void testGroupByCase() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        String tableName = generateUniqueName();
-        String groupBy1 = "select " +
-                "case when uri LIKE 'Report%' then 'Reports' else 'Other' END category" +
-                ", avg(appcpu) from " + tableName +
-                " group by category";
-
-        String groupBy2 = "select " +
-                "case uri when 'Report%' then 'Reports' else 'Other' END category" +
-                ", avg(appcpu) from " + tableName +
-                " group by appcpu, category";
-        
-        String groupBy3 = "select " +
-                "case uri when 'Report%' then 'Reports' else 'Other' END category" +
-                ", avg(appcpu) from " + tableName +
-                " group by avg(appcpu), category";
-        initData(conn, tableName);
-        conn.createStatement().executeQuery(groupBy1);
-        conn.createStatement().executeQuery(groupBy2);
-        // TODO: validate query results
-        try {
-            conn.createStatement().executeQuery(groupBy3);
-            fail();
-        } catch (SQLException e) {
-            assertTrue(e.getMessage().contains("Aggregate expressions may not be used in GROUP BY"));
-        }
-        conn.close();
-    }
-
-
-    @Test
-    public void testGroupByArray() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-
-        String tableName = generateUniqueName();
-        conn.createStatement().execute("CREATE TABLE " + tableName + "(\n" + 
-                "  a VARCHAR NOT NULL,\n" + 
-                "  b VARCHAR,\n" + 
-                "  c INTEGER,\n" + 
-                "  d VARCHAR,\n" + 
-                "  e VARCHAR ARRAY,\n" + 
-                "  f BIGINT,\n" + 
-                "  g BIGINT,\n" + 
-                "  CONSTRAINT pk PRIMARY KEY(a)\n" + 
-                ")");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('1', 'val', 100, 'a', ARRAY ['b'], 1, 2)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('2', 'val', 100, 'a', ARRAY ['b'], 3, 4)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('3', 'val', 100, 'a', ARRAY ['b','c'], 5, 6)");
-        conn.commit();
-        
-        ResultSet rs = conn.createStatement().executeQuery("SELECT c, SUM(f + g) AS sumone, d, e\n" + 
-                "FROM " + tableName + "\n" + 
-                "WHERE b = 'val'\n" + 
-                "  AND a IN ('1','2','3')\n" + 
-                "GROUP BY c, d, e\n" + 
-                "ORDER BY sumone DESC");
-        assertTrue(rs.next());
-        assertEquals(100, rs.getInt(1));
-        assertEquals(11, rs.getLong(2));
-        assertTrue(rs.next());
-        assertEquals(100, rs.getInt(1));
-        assertEquals(10, rs.getLong(2));
-        assertFalse(rs.next());
-        conn.close();
-    }
-    
-    @Test
-    public void testGroupByOrderPreserving() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        String tableName = generateUniqueName();
-
-        conn.createStatement().execute("CREATE TABLE " + tableName + "(ORGANIZATION_ID char(15) not null, \n" + 
-                "JOURNEY_ID char(15) not null, \n" + 
-                "DATASOURCE SMALLINT not null, \n" + 
-                "MATCH_STATUS TINYINT not null, \n" + 
-                "EXTERNAL_DATASOURCE_KEY varchar(30), \n" + 
-                "ENTITY_ID char(15) not null, \n" + 
-                "CONSTRAINT PK PRIMARY KEY (\n" + 
-                "    ORGANIZATION_ID, \n" + 
-                "    JOURNEY_ID, \n" + 
-                "    DATASOURCE, \n" + 
-                "    MATCH_STATUS,\n" + 
-                "    EXTERNAL_DATASOURCE_KEY,\n" + 
-                "    ENTITY_ID))");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('000001111122222', '333334444455555', 0, 0, 'abc', '666667777788888')");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('000001111122222', '333334444455555', 0, 0, 'abcd', '666667777788889')");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('000001111122222', '333334444455555', 0, 0, 'abc', '666667777788899')");
-        conn.commit();
-        String query =
-                "SELECT COUNT(1), EXTERNAL_DATASOURCE_KEY As DUP_COUNT\n" + 
-                "    FROM " + tableName + " \n" + 
-                "   WHERE JOURNEY_ID='333334444455555' AND \n" + 
-                "                 DATASOURCE=0 AND MATCH_STATUS <= 1 and \n" + 
-                "                 ORGANIZATION_ID='000001111122222' \n" + 
-                "    GROUP BY MATCH_STATUS, EXTERNAL_DATASOURCE_KEY \n" + 
-                "    HAVING COUNT(1) > 1";
-        ResultSet rs = conn.createStatement().executeQuery(query);
-        assertTrue(rs.next());
-        assertEquals(2,rs.getInt(1));
-        assertEquals("abc", rs.getString(2));
-        assertFalse(rs.next());
-        
-        rs = conn.createStatement().executeQuery("EXPLAIN " + query);
-        assertEquals(
-                "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName + " ['000001111122222','333334444455555',0,*] - ['000001111122222','333334444455555',0,1]\n" + 
-                "    SERVER FILTER BY FIRST KEY ONLY\n" + 
-                "    SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [MATCH_STATUS, EXTERNAL_DATASOURCE_KEY]\n" + 
-                "CLIENT FILTER BY COUNT(1) > 1",QueryUtil.getExplainPlan(rs));
-    }
-    
-    @Test
-    public void testGroupByOrderPreservingDescSort() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        String tableName = generateUniqueName(); 
-        conn.createStatement().execute("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 char(1) not null, constraint pk primary key (k1,k2)) split on ('ac','jc','nc')");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'a')");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'b')");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'c')");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'd')");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'a')");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'b')");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'c')");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'd')");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'a')");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'b')");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'c')");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'd')");
-        conn.commit();
-        String query = "SELECT k1,count(*) FROM " + tableName + " GROUP BY k1 ORDER BY k1 DESC";
-        ResultSet rs = conn.createStatement().executeQuery(query);
-        assertTrue(rs.next());
-        assertEquals("n", rs.getString(1));
-        assertEquals(4, rs.getInt(2));
-        assertTrue(rs.next());
-        assertEquals("j", rs.getString(1));
-        assertEquals(4, rs.getInt(2));
-        assertTrue(rs.next());
-        assertEquals("a", rs.getString(1));
-        assertEquals(4, rs.getInt(2));
-        assertFalse(rs.next());
-        rs = conn.createStatement().executeQuery("EXPLAIN " + query);
-        assertEquals(
-                "CLIENT PARALLEL 1-WAY REVERSE FULL SCAN OVER " + tableName + "\n" + 
-                "    SERVER FILTER BY FIRST KEY ONLY\n" + 
-                "    SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", QueryUtil.getExplainPlan(rs));
-    }
-    
-    @Test
-    public void testSumGroupByOrderPreservingDesc() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        String tableName = generateUniqueName();
-
-        PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) split on (?,?,?)");
-        stmt.setBytes(1, ByteUtil.concat(PChar.INSTANCE.toBytes("a"), PInteger.INSTANCE.toBytes(3)));
-        stmt.setBytes(2, ByteUtil.concat(PChar.INSTANCE.toBytes("j"), PInteger.INSTANCE.toBytes(3)));
-        stmt.setBytes(3, ByteUtil.concat(PChar.INSTANCE.toBytes("n"), PInteger.INSTANCE.toBytes(3)));
-        stmt.execute();
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 1)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 2)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 3)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 4)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('b', 5)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 1)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 2)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 3)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 4)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 1)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 4)");
-        conn.commit();
-        String query = "SELECT k1,sum(k2) FROM " + tableName + " GROUP BY k1 ORDER BY k1 DESC";
-        ResultSet rs = conn.createStatement().executeQuery(query);
-        assertTrue(rs.next());
-        assertEquals("n", rs.getString(1));
-        assertEquals(10, rs.getInt(2));
-        assertTrue(rs.next());
-        assertEquals("j", rs.getString(1));
-        assertEquals(10, rs.getInt(2));
-        assertTrue(rs.next());
-        assertEquals("b", rs.getString(1));
-        assertEquals(5, rs.getInt(2));
-        assertTrue(rs.next());
-        assertEquals("a", rs.getString(1));
-        assertEquals(10, rs.getInt(2));
-        assertFalse(rs.next());
-        rs = conn.createStatement().executeQuery("EXPLAIN " + query);
-        assertEquals(
-                "CLIENT PARALLEL 1-WAY REVERSE FULL SCAN OVER " + tableName + "\n" + 
-                "    SERVER FILTER BY FIRST KEY ONLY\n" + 
-                "    SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", QueryUtil.getExplainPlan(rs));
-    }
-
-    @Test
-    public void testAvgGroupByOrderPreservingWithStats() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        String tableName = generateUniqueName();
-        ResultSet rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + "\"SYSTEM\".\"STATS\"" + " WHERE " + PhoenixDatabaseMetaData.PHYSICAL_NAME + " ='" + tableName + "'");
-        assertTrue(rs.next());
-        assertEquals(0,rs.getInt(1));
-        initAvgGroupTable(conn, tableName, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=20 ");
-        testAvgGroupByOrderPreserving(conn, tableName, 13);
-        rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + "\"SYSTEM\".\"STATS\"" + " WHERE " + PhoenixDatabaseMetaData.PHYSICAL_NAME + " ='" + tableName + "'");
-        assertTrue(rs.next());
-        assertEquals(13,rs.getInt(1));
-        conn.setAutoCommit(true);
-        conn.createStatement().execute("DELETE FROM " + "\"SYSTEM\".\"STATS\"");
-        rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + "\"SYSTEM\".\"STATS\"" + " WHERE " + PhoenixDatabaseMetaData.PHYSICAL_NAME + " ='" + tableName + "'");
-        assertTrue(rs.next());
-        assertEquals(0,rs.getInt(1));
-        TestUtil.doMajorCompaction(conn, tableName);
-        rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + "\"SYSTEM\".\"STATS\"" + " WHERE " + PhoenixDatabaseMetaData.PHYSICAL_NAME + " ='" + tableName + "'");
-        assertTrue(rs.next());
-        assertEquals(13,rs.getInt(1));
-        testAvgGroupByOrderPreserving(conn, tableName, 13);
-        conn.createStatement().execute("ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=100");
-        testAvgGroupByOrderPreserving(conn, tableName, 6);
-        conn.createStatement().execute("ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=null");
-        testAvgGroupByOrderPreserving(conn, tableName, 4);
-    }
-    
-    @Test
-    public void testAvgGroupByOrderPreservingWithNoStats() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        String tableName = generateUniqueName();
-        initAvgGroupTable(conn, tableName, "");
-        testAvgGroupByOrderPreserving(conn, tableName, 4);
-    }
-    
-    private void initAvgGroupTable(Connection conn, String tableName, String tableProps) throws SQLException {
-        PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) " + tableProps + " split on (?,?,?)");
-        stmt.setBytes(1, ByteUtil.concat(PChar.INSTANCE.toBytes("a"), PInteger.INSTANCE.toBytes(3)));
-        stmt.setBytes(2, ByteUtil.concat(PChar.INSTANCE.toBytes("j"), PInteger.INSTANCE.toBytes(3)));
-        stmt.setBytes(3, ByteUtil.concat(PChar.INSTANCE.toBytes("n"), PInteger.INSTANCE.toBytes(3)));
-        stmt.execute();
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 1)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 2)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 3)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 6)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('b', 5)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 1)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 2)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 3)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 10)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 1)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
-        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
-        conn.commit();
-    }
-    
-    private void testAvgGroupByOrderPreserving(Connection conn, String tableName, int nGuidePosts) throws SQLException, IOException {
-        String query = "SELECT k1,avg(k2) FROM " + tableName + " GROUP BY k1";
-        ResultSet rs = conn.createStatement().executeQuery(query);
-        assertTrue(rs.next());
-        assertEquals("a", rs.getString(1));
-        assertEquals(3, rs.getInt(2));
-        assertTrue(rs.next());
-        assertEquals("b", rs.getString(1));
-        assertEquals(5, rs.getInt(2));
-        assertTrue(rs.next());
-        assertEquals("j", rs.getString(1));
-        assertEquals(4, rs.getInt(2));
-        assertTrue(rs.next());
-        assertEquals("n", rs.getString(1));
-        assertEquals(2, rs.getInt(2));
-        assertFalse(rs.next());
-        rs = conn.createStatement().executeQuery("EXPLAIN " + query);
-        assertEquals(
-                "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + tableName + "\n" + 
-                "    SERVER FILTER BY FIRST KEY ONLY\n" + 
-                "    SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", QueryUtil.getExplainPlan(rs));
-        TestUtil.analyzeTable(conn, tableName);
-        List<KeyRange> splits = TestUtil.getAllSplits(conn, tableName);
-        assertEquals(nGuidePosts, splits.size());
-    }
-    
     @Test
     public void testGroupByWithAliasWithSameColumnName() throws SQLException {
         Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
@@ -557,344 +79,6 @@ public class AggregateIT extends ParallelStatsDisabledIT {
     }
 
     @Test
-    public void testDistinctGroupByBug3452WithoutMultiTenant() throws Exception {
-        doTestDistinctGroupByBug3452("");
-    }
-
-    @Test
-    public void testDistinctGroupByBug3452WithMultiTenant() throws Exception {
-        doTestDistinctGroupByBug3452("VERSIONS=1, MULTI_TENANT=TRUE, REPLICATION_SCOPE=1, TTL=31536000");
-    }
-
-    private void doTestDistinctGroupByBug3452(String options) throws Exception {
-        Connection conn=null;
-        try {
-            Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-            conn = DriverManager.getConnection(getUrl(), props);
-
-            String tableName=generateUniqueName();
-            conn.createStatement().execute("DROP TABLE if exists "+tableName);
-            String sql="CREATE TABLE "+ tableName +" ( "+
-                    "ORGANIZATION_ID CHAR(15) NOT NULL,"+
-                    "CONTAINER_ID CHAR(15) NOT NULL,"+
-                    "ENTITY_ID CHAR(15) NOT NULL,"+
-                    "SCORE DOUBLE,"+
-                    "CONSTRAINT TEST_PK PRIMARY KEY ( "+
-                    "ORGANIZATION_ID,"+
-                    "CONTAINER_ID,"+
-                    "ENTITY_ID"+
-                    ")) "+options;
-            conn.createStatement().execute(sql);
-
-            String indexTableName=generateUniqueName();
-            conn.createStatement().execute("DROP INDEX IF EXISTS "+indexTableName+" ON "+tableName);
-            conn.createStatement().execute("CREATE INDEX "+indexTableName+" ON "+tableName+" (CONTAINER_ID, SCORE DESC, ENTITY_ID DESC)");
-
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId6',1.1)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId5',1.2)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId4',1.3)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId3',1.4)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId2',1.5)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId1',1.6)");
-            conn.commit();
-
-            sql="SELECT DISTINCT entity_id,score FROM "+tableName+" WHERE organization_id = 'org1' AND container_id = 'container1' ORDER BY score DESC";
-            ResultSet rs=conn.createStatement().executeQuery(sql);
-            assertTrue(rs.next());
-            assertTrue(rs.getString(1).equals("entityId1"));
-            assertEquals(rs.getDouble(2),1.6,0.0001);
-
-            assertTrue(rs.next());
-            assertTrue(rs.getString(1).equals("entityId2"));
-            assertEquals(rs.getDouble(2),1.5,0.0001);
-
-            assertTrue(rs.next());
-            assertTrue(rs.getString(1).equals("entityId3"));
-            assertEquals(rs.getDouble(2),1.4,0.0001);
-
-            assertTrue(rs.next());
-            assertTrue(rs.getString(1).equals("entityId4"));
-            assertEquals(rs.getDouble(2),1.3,0.0001);
-
-            assertTrue(rs.next());
-            assertTrue(rs.getString(1).equals("entityId5"));
-            assertEquals(rs.getDouble(2),1.2,0.0001);
-
-            assertTrue(rs.next());
-            assertTrue(rs.getString(1).equals("entityId6"));
-            assertEquals(rs.getDouble(2),1.1,0.0001);
-            assertTrue(!rs.next());
-        } finally {
-            if(conn!=null) {
-                conn.close();
-            }
-        }
-    }
-
-    @Test
-    public void testGroupByOrderByDescBug3451() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
-            String tableName=generateUniqueName();
-            String sql="CREATE TABLE " + tableName + " (\n" + 
-                    "            ORGANIZATION_ID CHAR(15) NOT NULL,\n" + 
-                    "            CONTAINER_ID CHAR(15) NOT NULL,\n" + 
-                    "            ENTITY_ID CHAR(15) NOT NULL,\n" + 
-                    "            SCORE DOUBLE,\n" + 
-                    "            CONSTRAINT TEST_PK PRIMARY KEY (\n" + 
-                    "               ORGANIZATION_ID,\n" + 
-                    "               CONTAINER_ID,\n" + 
-                    "               ENTITY_ID\n" + 
-                    "             )\n" + 
-                    "         )";
-            conn.createStatement().execute(sql);
-            String indexName=generateUniqueName();
-            conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + "(ORGANIZATION_ID,CONTAINER_ID, SCORE DESC, ENTITY_ID DESC)");
-            conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container2','entityId6',1.1)");
-            conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container1','entityId5',1.2)");
-            conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container2','entityId4',1.3)");
-            conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container1','entityId5',1.2)");
-            conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container1','entityId3',1.4)");
-            conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container3','entityId7',1.35)");
-            conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container3','entityId8',1.45)");
-            conn.commit();
-            String query = "SELECT DISTINCT entity_id, score\n" + 
-                    "    FROM " + tableName + "\n" +
-                    "    WHERE organization_id = 'org2'\n" + 
-                    "    AND container_id IN ( 'container1','container2','container3' )\n" + 
-                    "    ORDER BY score DESC\n" + 
-                    "    LIMIT 2";
-            Statement stmt = conn.createStatement();
-            ResultSet rs = stmt.executeQuery(query);
-            QueryPlan plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
-            assertEquals(indexName, plan.getContext().getCurrentTable().getTable().getName().getString());
-            assertFalse(plan.getOrderBy().getOrderByExpressions().isEmpty());
-            assertTrue(rs.next());
-            assertEquals("entityId8", rs.getString(1));
-            assertEquals(1.45, rs.getDouble(2),0.001);
-            assertTrue(rs.next());
-            assertEquals("entityId3", rs.getString(1));
-            assertEquals(1.4, rs.getDouble(2),0.001);
-            assertFalse(rs.next());
-       }
-    }
-    
-    @Test
-    public void testGroupByDescColumnWithNullsLastBug3452() throws Exception {
-
-        Connection conn=null;
-        try
-        {
-            Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-            conn = DriverManager.getConnection(getUrl(), props);
-
-            String tableName=generateUniqueName();
-            String sql="CREATE TABLE "+tableName+" ( "+
-                    "ORGANIZATION_ID VARCHAR,"+
-                    "CONTAINER_ID VARCHAR,"+
-                    "ENTITY_ID VARCHAR NOT NULL,"+
-                    "CONSTRAINT TEST_PK PRIMARY KEY ( "+
-                    "ORGANIZATION_ID DESC,"+
-                    "CONTAINER_ID DESC,"+
-                    "ENTITY_ID"+
-                    "))";
-            conn.createStatement().execute(sql);
-
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('a',null,'11')");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'2','22')");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('c','3','33')");
-            conn.commit();
-
-            //-----ORGANIZATION_ID
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID ASC NULLS FIRST";
-            ResultSet rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{null,"a"},{"3","c"},});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null}});
-
-            //----CONTAINER_ID
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID ASC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{"2",null},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"}});
-
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,'44')");
-            conn.commit();
-
-            //-----ORGANIZATION_ID ASC  CONTAINER_ID ASC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
-
-            //-----ORGANIZATION_ID ASC  CONTAINER_ID DESC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
-
-            //-----ORGANIZATION_ID DESC  CONTAINER_ID ASC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
-
-            //-----ORGANIZATION_ID DESC  CONTAINER_ID DESC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
-
-            //-----CONTAINER_ID ASC  ORGANIZATION_ID ASC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
-
-            //-----CONTAINER_ID ASC  ORGANIZATION_ID DESC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID  order by CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
-
-            //-----CONTAINER_ID DESC  ORGANIZATION_ID ASC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
-
-            //-----CONTAINER_ID DESC  ORGANIZATION_ID DESC
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
-
-            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST";
-            rs=conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
-        } finally {
-            if(conn!=null) {
-                conn.close();
-            }
-        }
-    }
-
-    @Test
     public void testGroupByCoerceExpressionBug3453() throws Exception {
         final Connection conn = DriverManager.getConnection(getUrl());
         try {
@@ -955,16 +139,6 @@ public class AggregateIT extends ParallelStatsDisabledIT {
     }
 
     @Test
-    public void testCountNullInEncodedNonEmptyKeyValueCF() throws Exception {
-        testCountNullInNonEmptyKeyValueCF(1);
-    }
-    
-    @Test
-    public void testCountNullInNonEncodedNonEmptyKeyValueCF() throws Exception {
-        testCountNullInNonEmptyKeyValueCF(0);
-    }
-
-    @Test
     public void testNestedGroupedAggregationWithBigInt() throws Exception {
         Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
         String tableName = generateUniqueName();
@@ -983,7 +157,41 @@ public class AggregateIT extends ParallelStatsDisabledIT {
         }
     }
 
-    private void testCountNullInNonEmptyKeyValueCF(int columnEncodedBytes) throws Exception {
+    @Test
+    public void testAvgGroupByOrderPreservingWithStats() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String tableName = generateUniqueName();
+        QueryBuilder queryBuilder = new QueryBuilder()
+            .setSelectExpression("COUNT(*)")
+            .setFullTableName(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)
+            .setWhereClause(PhoenixDatabaseMetaData.PHYSICAL_NAME + " ='" + tableName + "'");
+        ResultSet rs = executeQuery(conn, queryBuilder);
+        assertTrue(rs.next());
+        assertEquals(0,rs.getInt(1));
+        initAvgGroupTable(conn, tableName, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=20 ");
+        testAvgGroupByOrderPreserving(conn, tableName, 13);
+        rs = executeQuery(conn, queryBuilder);
+        assertTrue(rs.next());
+        assertEquals(13,rs.getInt(1));
+        conn.setAutoCommit(true);
+        conn.createStatement().execute("DELETE FROM " + "\"SYSTEM\".\"STATS\"");
+        rs = executeQuery(conn, queryBuilder);
+        assertTrue(rs.next());
+        assertEquals(0,rs.getInt(1));
+        TestUtil.doMajorCompaction(conn, tableName);
+        rs = executeQuery(conn, queryBuilder);
+        assertTrue(rs.next());
+        assertEquals(13,rs.getInt(1));
+        testAvgGroupByOrderPreserving(conn, tableName, 13);
+        conn.createStatement().execute("ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=100");
+        testAvgGroupByOrderPreserving(conn, tableName, 6);
+        conn.createStatement().execute("ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=null");
+        testAvgGroupByOrderPreserving(conn, tableName, 4);
+    }
+
+    @Override
+    protected void testCountNullInNonEmptyKeyValueCF(int columnEncodedBytes) throws Exception {
         try (Connection conn = DriverManager.getConnection(getUrl())) {
             //Type is INT
             String intTableName=generateUniqueName();
@@ -998,119 +206,26 @@ public class AggregateIT extends ParallelStatsDisabledIT {
             conn.createStatement().execute("UPSERT INTO "+intTableName+" VALUES (5,1)");
             conn.commit();
 
-            TestUtil.dumpTable(conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes(intTableName)));
-
             sql="select count(*) from "+intTableName;
-            ResultSet rs=conn.createStatement().executeQuery(sql);
+            QueryBuilder queryBuilder = new QueryBuilder()
+                .setSelectExpression("COUNT(*)")
+                .setFullTableName(intTableName);
+            ResultSet rs = executeQuery(conn, queryBuilder);
             assertTrue(rs.next());
-            assertEquals(5, rs.getInt(1));
-            
+            assertEquals(5, rs.getLong(1));
+
             sql="select count(*) from "+intTableName + " where b.colb is not null";
-            rs=conn.createStatement().executeQuery(sql);
+            queryBuilder.setWhereClause("B.COLB IS NOT NULL");
+            rs = executeQuery(conn, queryBuilder);
             assertTrue(rs.next());
-            assertEquals(1, rs.getInt(1));
+            assertEquals(1, rs.getLong(1));
 
             sql="select count(*) from "+intTableName + " where b.colb is null";
-            rs=conn.createStatement().executeQuery(sql);
+            queryBuilder.setWhereClause("B.COLB IS NULL");
+            rs = executeQuery(conn, queryBuilder);
             assertTrue(rs.next());
-            assertEquals(4, rs.getInt(1));
-        }
-    }
-
-    @Test
-    public void testGroupByOrderMatchPkColumnOrderBug4690() throws Exception {
-        this.doTestGroupByOrderMatchPkColumnOrderBug4690(false, false);
-        this.doTestGroupByOrderMatchPkColumnOrderBug4690(false, true);
-        this.doTestGroupByOrderMatchPkColumnOrderBug4690(true, false);
-        this.doTestGroupByOrderMatchPkColumnOrderBug4690(true, true);
-    }
-
-    private void doTestGroupByOrderMatchPkColumnOrderBug4690(boolean desc ,boolean salted) throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = null;
-        try {
-            conn = DriverManager.getConnection(getUrl(), props);
-            String tableName = generateUniqueName();
-            String sql = "create table " + tableName + "( "+
-                    " pk1 integer not null , " +
-                    " pk2 integer not null, " +
-                    " pk3 integer not null," +
-                    " pk4 integer not null,"+
-                    " v integer, " +
-                    " CONSTRAINT TEST_PK PRIMARY KEY ( "+
-                       "pk1 "+(desc ? "desc" : "")+", "+
-                       "pk2 "+(desc ? "desc" : "")+", "+
-                       "pk3 "+(desc ? "desc" : "")+", "+
-                       "pk4 "+(desc ? "desc" : "")+
-                    " )) "+(salted ? "SALT_BUCKETS =4" : "split on(2)");
-            conn.createStatement().execute(sql);
-
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,8,10,20,30)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,8,11,21,31)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,5 ,22,32)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,6 ,12,33)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,6 ,13,34)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,7 ,8,35)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,3,15,25,35)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,7,16,26,36)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,7,17,27,37)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,2,18,28,38)");
-            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,2,19,29,39)");
-            conn.commit();
-
-            sql = "select pk2,pk1,count(v) from " + tableName + " group by pk2,pk1 order by pk2,pk1";
-            ResultSet rs = conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{2,3,2L},{3,2,1L},{7,2,2L},{8,1,2L},{9,1,4L}});
-
-            sql = "select pk1,pk2,count(v) from " + tableName + " group by pk2,pk1 order by pk1,pk2";
-            rs = conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{1,8,2L},{1,9,4L},{2,3,1L},{2,7,2L},{3,2,2L}});
-
-            sql = "select pk2,pk1,count(v) from " + tableName + " group by pk2,pk1 order by pk2 desc,pk1 desc";
-            rs = conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{9,1,4L},{8,1,2L},{7,2,2L},{3,2,1L},{2,3,2L}});
-
-            sql = "select pk1,pk2,count(v) from " + tableName + " group by pk2,pk1 order by pk1 desc,pk2 desc";
-            rs = conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{3,2,2L},{2,7,2L},{2,3,1L},{1,9,4L},{1,8,2L}});
-
-
-            sql = "select pk3,pk2,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk3,pk2";
-            rs = conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{5,9,1L},{6,9,2L},{7,9,1L},{10,8,1L},{11,8,1L}});
-
-            sql = "select pk2,pk3,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk2,pk3";
-            rs = conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{8,10,1L},{8,11,1L},{9,5,1L},{9,6,2L},{9,7,1L}});
-
-            sql = "select pk3,pk2,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk3 desc,pk2 desc";
-            rs = conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{11,8,1L},{10,8,1L},{7,9,1L},{6,9,2L},{5,9,1L}});
-
-            sql = "select pk2,pk3,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk2 desc,pk3 desc";
-            rs = conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{9,7,1L},{9,6,2L},{9,5,1L},{8,11,1L},{8,10,1L}});
-
-
-            sql = "select pk4,pk3,pk1,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk4,pk3,pk1";
-            rs = conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{8,7,1,1L},{12,6,1,1L},{13,6,1,1L},{22,5,1,1L}});
-
-            sql = "select pk1,pk3,pk4,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk1,pk3,pk4";
-            rs = conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{1,5,22,1L},{1,6,12,1L},{1,6,13,1L},{1,7,8,1L}});
-
-            sql = "select pk4,pk3,pk1,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk4 desc,pk3 desc,pk1 desc";
-            rs = conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{22,5,1,1L},{13,6,1,1L},{12,6,1,1L},{8,7,1,1L}});
-
-            sql = "select pk1,pk3,pk4,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk1 desc,pk3 desc,pk4 desc";
-            rs = conn.prepareStatement(sql).executeQuery();
-            assertResultSet(rs, new Object[][]{{1,7,8,1L},{1,6,13,1L},{1,6,12,1L},{1,5,22,1L}});
-        } finally {
-            if(conn != null) {
-                conn.close();
-            }
+            assertEquals(4, rs.getLong(1));
         }
     }
 }
+


[19/22] phoenix git commit: PHOENIX-5012 Don't derive IndexToolIT from ParallelStatsEnabled

Posted by ja...@apache.org.
PHOENIX-5012 Don't derive IndexToolIT from ParallelStatsEnabled


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d1efd665
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d1efd665
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d1efd665

Branch: refs/heads/omid2
Commit: d1efd665fa84f611e1ad544ed0859f1fecb4f616
Parents: fd7ae94
Author: James Taylor <ja...@apache.org>
Authored: Sat Nov 10 11:04:48 2018 -0800
Committer: James Taylor <ja...@apache.org>
Committed: Sat Nov 10 11:04:48 2018 -0800

----------------------------------------------------------------------
 .../src/it/java/org/apache/phoenix/end2end/IndexToolIT.java  | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/d1efd665/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
index c99f145..e096bb5 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
@@ -58,7 +58,6 @@ import org.apache.phoenix.util.SchemaUtil;
 import org.apache.phoenix.util.TestUtil;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
@@ -67,8 +66,7 @@ import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
 @RunWith(Parameterized.class)
-@Category(NeedsOwnMiniClusterTest.class)
-public class IndexToolIT extends ParallelStatsEnabledIT {
+public class IndexToolIT extends BaseUniqueNamesOwnClusterIT {
 
     private final boolean localIndex;
     private final boolean transactional;
@@ -99,9 +97,13 @@ public class IndexToolIT extends ParallelStatsEnabledIT {
     @BeforeClass
     public static void setup() throws Exception {
         Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2);
+        serverProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
+        serverProps.put(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB, Long.toString(5));
         serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB,
             QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
         Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
+        clientProps.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(true));
+        clientProps.put(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5));
         clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
         clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.TRUE.toString());
         setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()),


[17/22] phoenix git commit: PHOENIX-4996: Refactor PTableImpl to use Builder Pattern

Posted by ja...@apache.org.
PHOENIX-4996: Refactor PTableImpl to use Builder Pattern


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d6083ae5
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d6083ae5
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d6083ae5

Branch: refs/heads/omid2
Commit: d6083ae5c598f5293adbfb24370b8e40962bc3d7
Parents: 9bfaf18
Author: Chinmay Kulkarni <ch...@gmail.com>
Authored: Fri Nov 2 14:00:09 2018 -0700
Committer: Thomas D'Silva <td...@apache.org>
Committed: Tue Nov 6 15:19:09 2018 -0800

----------------------------------------------------------------------
 .../apache/phoenix/compile/DeleteCompiler.java  |    5 +-
 .../apache/phoenix/compile/FromCompiler.java    |   66 +-
 .../apache/phoenix/compile/JoinCompiler.java    |   53 +-
 .../compile/TupleProjectionCompiler.java        |   60 +-
 .../apache/phoenix/compile/UnionCompiler.java   |   41 +-
 .../apache/phoenix/compile/UpsertCompiler.java  |   12 +-
 .../coprocessor/MetaDataEndpointImpl.java       |   96 +-
 .../UngroupedAggregateRegionObserver.java       |    6 +-
 .../coprocessor/WhereConstantParser.java        |    3 +-
 .../query/ConnectionlessQueryServicesImpl.java  |    9 +-
 .../apache/phoenix/schema/MetaDataClient.java   |  215 ++-
 .../apache/phoenix/schema/PMetaDataImpl.java    |   28 +-
 .../org/apache/phoenix/schema/PTableImpl.java   | 1259 +++++++++++-------
 .../org/apache/phoenix/schema/TableRef.java     |   17 +-
 .../phoenix/execute/CorrelatePlanTest.java      |   32 +-
 .../execute/LiteralResultIteratorPlanTest.java  |   33 +-
 16 files changed, 1303 insertions(+), 632 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
index 583085e..8c9a930 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
@@ -89,7 +89,6 @@ import org.apache.phoenix.schema.types.PLong;
 import org.apache.phoenix.transaction.PhoenixTransactionProvider.Feature;
 import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.IndexUtil;
-import org.apache.phoenix.util.MetaDataUtil;
 import org.apache.phoenix.util.ScanUtil;
 
 import com.google.common.base.Preconditions;
@@ -615,7 +614,9 @@ public class DeleteCompiler {
                     }
                 });
             }
-            PTable projectedTable = PTableImpl.makePTable(table, PTableType.PROJECTED, adjustedProjectedColumns);
+            PTable projectedTable = PTableImpl.builderWithColumns(table, adjustedProjectedColumns)
+                    .setType(PTableType.PROJECTED)
+                    .build();
             final TableRef projectedTableRef = new TableRef(projectedTable, targetTableRef.getLowerBoundTimeStamp(), targetTableRef.getTimeStamp());
 
             QueryPlan bestPlanToBe = dataPlan;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
index efc66a9..2701af0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
@@ -32,8 +32,6 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.coprocessor.MetaDataProtocol;
 import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
 import org.apache.phoenix.coprocessor.MetaDataProtocol.MutationCode;
-import org.apache.phoenix.exception.SQLExceptionCode;
-import org.apache.phoenix.exception.SQLExceptionInfo;
 import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.parse.AliasedNode;
@@ -82,6 +80,7 @@ import org.apache.phoenix.schema.PTableImpl;
 import org.apache.phoenix.schema.PTableKey;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.ProjectedColumn;
+import org.apache.phoenix.schema.RowKeySchema;
 import org.apache.phoenix.schema.SchemaNotFoundException;
 import org.apache.phoenix.schema.SortOrder;
 import org.apache.phoenix.schema.TableNotFoundException;
@@ -284,7 +283,8 @@ public class FromCompiler {
                 column.getTimestamp());
             projectedColumns.add(projectedColumn);
         }
-        PTable t = PTableImpl.makePTable(table, projectedColumns);
+        PTable t = PTableImpl.builderWithColumns(table, projectedColumns)
+                .build();
         return new SingleTableColumnResolver(connection, new TableRef(tableRef.getTableAlias(), t, tableRef.getLowerBoundTimeStamp(), tableRef.hasDynamicCols()));
     }
     
@@ -369,10 +369,30 @@ public class FromCompiler {
             if (connection.getSchema() != null) {
                 schema = schema != null ? schema : connection.getSchema();
             }
+
             // Storage scheme and encoding scheme don't matter here since the PTable is being used only for the purposes of create table.
             // The actual values of these two will be determined by the metadata client.
-            PTable theTable = new PTableImpl(connection.getTenantId(), schema, table.getName().getTableName(),
-                    scn == null ? HConstants.LATEST_TIMESTAMP : scn, families, isNamespaceMapped);
+            PName tenantId = connection.getTenantId();
+            PTableImpl.checkTenantId(tenantId);
+            String tableName = table.getName().getTableName();
+            PName name = PNameFactory.newName(SchemaUtil.getTableName(schema, tableName));
+            PTable theTable = new PTableImpl.Builder()
+                    .setTenantId(tenantId)
+                    .setName(name)
+                    .setKey(new PTableKey(tenantId, name.getString()))
+                    .setSchemaName(PNameFactory.newName(schema))
+                    .setTableName(PNameFactory.newName(tableName))
+                    .setType(PTableType.VIEW)
+                    .setViewType(PTable.ViewType.MAPPED)
+                    .setTimeStamp(scn == null ? HConstants.LATEST_TIMESTAMP : scn)
+                    .setPkColumns(Collections.<PColumn>emptyList())
+                    .setAllColumns(Collections.<PColumn>emptyList())
+                    .setRowKeySchema(RowKeySchema.EMPTY_SCHEMA)
+                    .setIndexes(Collections.<PTable>emptyList())
+                    .setFamilyAttributes(families)
+                    .setPhysicalNames(Collections.<PName>emptyList())
+                    .setNamespaceMapped(isNamespaceMapped)
+                    .build();
             theTable = this.addDynamicColumns(table.getDynamicColumns(), theTable);
             alias = null;
             tableRefs = ImmutableList.of(new TableRef(alias, theTable, timeStamp, !table.getDynamicColumns().isEmpty()));
@@ -706,7 +726,7 @@ public class FromCompiler {
             if (!dynColumns.isEmpty()) {
                 List<PColumn> allcolumns = new ArrayList<PColumn>();
                 List<PColumn> existingColumns = theTable.getColumns();
-                // Need to skip the salting column, as it's added in the makePTable call below
+                // Need to skip the salting column, as it's handled in the PTable builder call below
                 allcolumns.addAll(theTable.getBucketNum() == null ? existingColumns : existingColumns.subList(1, existingColumns.size()));
                 // Position still based on with the salting columns
                 int position = existingColumns.size();
@@ -724,7 +744,8 @@ public class FromCompiler {
                         HConstants.LATEST_TIMESTAMP));
                     position++;
                 }
-                theTable = PTableImpl.makePTable(theTable, allcolumns);
+                theTable = PTableImpl.builderWithColumns(theTable, allcolumns)
+                        .build();
             }
             return theTable;
         }
@@ -830,11 +851,32 @@ public class FromCompiler {
                     HConstants.LATEST_TIMESTAMP);
                 columns.add(column);
             }
-            PTable t = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME, PTableType.SUBQUERY, null,
-                    MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null,
-                    Collections.<PTable> emptyList(), false, Collections.<PName> emptyList(), null, null, false, false,
-                    false, null, null, null, null, false, null, 0, 0L, SchemaUtil
-                            .isNamespaceMappingEnabled(PTableType.SUBQUERY, connection.getQueryServices().getProps()), null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, true);
+            PTable t = new PTableImpl.Builder()
+                    .setType(PTableType.SUBQUERY)
+                    .setTimeStamp(MetaDataProtocol.MIN_TABLE_TIMESTAMP)
+                    .setIndexDisableTimestamp(0L)
+                    .setSequenceNumber(PTable.INITIAL_SEQ_NUM)
+                    .setImmutableRows(false)
+                    .setDisableWAL(false)
+                    .setMultiTenant(false)
+                    .setStoreNulls(false)
+                    .setUpdateCacheFrequency(0)
+                    .setNamespaceMapped(SchemaUtil.isNamespaceMappingEnabled(PTableType.SUBQUERY,
+                            connection.getQueryServices().getProps()))
+                    .setAppendOnlySchema(false)
+                    .setImmutableStorageScheme(ImmutableStorageScheme.ONE_CELL_PER_COLUMN)
+                    .setQualifierEncodingScheme(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS)
+                    .setBaseColumnCount(QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT)
+                    .setEncodedCQCounter(PTable.EncodedCQCounter.NULL_COUNTER)
+                    .setUseStatsForParallelization(true)
+                    .setExcludedColumns(ImmutableList.<PColumn>of())
+                    .setSchemaName(PName.EMPTY_NAME)
+                    .setTableName(PName.EMPTY_NAME)
+                    .setRowKeyOrderOptimizable(false)
+                    .setIndexes(Collections.<PTable>emptyList())
+                    .setPhysicalNames(ImmutableList.<PName>of())
+                    .setColumns(columns)
+                    .build();
 
             String alias = subselectNode.getAlias();
             TableRef tableRef = new TableRef(alias, t, MetaDataProtocol.MIN_TABLE_TIMESTAMP, false);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
index 880fa72..807e54b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
@@ -17,6 +17,7 @@
  */
 package org.apache.phoenix.compile;
 
+import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
 import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN;
 import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
 
@@ -32,6 +33,7 @@ import java.util.ListIterator;
 import java.util.Map;
 import java.util.Set;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.phoenix.exception.SQLExceptionCode;
@@ -71,8 +73,8 @@ import org.apache.phoenix.schema.ColumnRef;
 import org.apache.phoenix.schema.LocalIndexDataColumnRef;
 import org.apache.phoenix.schema.MetaDataEntityNotFoundException;
 import org.apache.phoenix.schema.PColumn;
-import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PNameFactory;
+import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTable.IndexType;
 import org.apache.phoenix.schema.PTableImpl;
@@ -1270,17 +1272,44 @@ public class JoinCompiler {
         if (left.getBucketNum() != null) {
             merged.remove(0);
         }
-        return PTableImpl.makePTable(left.getTenantId(), left.getSchemaName(),
-                PNameFactory.newName(SchemaUtil.getTableName(left.getName().getString(), right.getName().getString())),
-                left.getType(), left.getIndexState(), left.getTimeStamp(), left.getSequenceNumber(), left.getPKName(),
-                left.getBucketNum(), merged, left.getParentSchemaName(), left.getParentTableName(), left.getIndexes(),
-                left.isImmutableRows(), Collections.<PName> emptyList(), null, null, PTable.DEFAULT_DISABLE_WAL,
-                left.isMultiTenant(), left.getStoreNulls(), left.getViewType(), left.getViewIndexType(), left.getViewIndexId(),
-                left.getIndexType(), left.rowKeyOrderOptimizable(), left.getTransactionProvider(),
-                left.getUpdateCacheFrequency(), left.getIndexDisableTimestamp(), left.isNamespaceMapped(), 
-                left.getAutoPartitionSeqName(), left.isAppendOnlySchema(), ONE_CELL_PER_COLUMN, NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, left.useStatsForParallelization());
+        return new PTableImpl.Builder()
+                .setType(left.getType())
+                .setState(left.getIndexState())
+                .setTimeStamp(left.getTimeStamp())
+                .setIndexDisableTimestamp(left.getIndexDisableTimestamp())
+                .setSequenceNumber(left.getSequenceNumber())
+                .setImmutableRows(left.isImmutableRows())
+                .setDisableWAL(PTable.DEFAULT_DISABLE_WAL)
+                .setMultiTenant(left.isMultiTenant())
+                .setStoreNulls(left.getStoreNulls())
+                .setViewType(left.getViewType())
+                .setViewIndexType(left.getViewIndexType())
+                .setViewIndexId(left.getViewIndexId())
+                .setIndexType(left.getIndexType())
+                .setTransactionProvider(left.getTransactionProvider())
+                .setUpdateCacheFrequency(left.getUpdateCacheFrequency())
+                .setNamespaceMapped(left.isNamespaceMapped())
+                .setAutoPartitionSeqName(left.getAutoPartitionSeqName())
+                .setAppendOnlySchema(left.isAppendOnlySchema())
+                .setImmutableStorageScheme(ONE_CELL_PER_COLUMN)
+                .setQualifierEncodingScheme(NON_ENCODED_QUALIFIERS)
+                .setBaseColumnCount(BASE_TABLE_BASE_COLUMN_COUNT)
+                .setEncodedCQCounter(PTable.EncodedCQCounter.NULL_COUNTER)
+                .setUseStatsForParallelization(left.useStatsForParallelization())
+                .setExcludedColumns(ImmutableList.<PColumn>of())
+                .setTenantId(left.getTenantId())
+                .setSchemaName(left.getSchemaName())
+                .setTableName(PNameFactory.newName(SchemaUtil.getTableName(left.getName().getString(),
+                        right.getName().getString())))
+                .setPkName(left.getPKName())
+                .setRowKeyOrderOptimizable(left.rowKeyOrderOptimizable())
+                .setBucketNum(left.getBucketNum())
+                .setIndexes(left.getIndexes() == null ? Collections.<PTable>emptyList() : left.getIndexes())
+                .setParentSchemaName(left.getParentSchemaName())
+                .setParentTableName(left.getParentTableName())
+                .setPhysicalNames(ImmutableList.<PName>of())
+                .setColumns(merged)
+                .build();
     }
 
 }
-
-

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
index dc85bd3..5b92a5d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
@@ -17,6 +17,8 @@
  */
 package org.apache.phoenix.compile;
 import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY;
+import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
+import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 
 import java.sql.SQLException;
 import java.util.ArrayList;
@@ -26,6 +28,7 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.phoenix.parse.AliasedNode;
 import org.apache.phoenix.parse.ColumnParseNode;
 import org.apache.phoenix.parse.FamilyWildcardParseNode;
@@ -160,15 +163,13 @@ public class TupleProjectionCompiler {
                     sourceColumnRef.getColumn().isNullable(), sourceColumnRef, sourceColumnRef.getColumn().getColumnQualifierBytes());
             projectedColumns.add(column);
         }
-        
-        return PTableImpl.makePTable(table.getTenantId(), table.getSchemaName(), table.getTableName(),
-                PTableType.PROJECTED, table.getIndexState(), table.getTimeStamp(), table.getSequenceNumber(),
-                table.getPKName(), table.getBucketNum(), projectedColumns, table.getParentSchemaName(),
-                table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), Collections.<PName> emptyList(),
-                table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(),
-                table.getViewIndexType(), table.getViewIndexId(),
-                table.getIndexType(), table.rowKeyOrderOptimizable(), table.getTransactionProvider(), table.getUpdateCacheFrequency(), 
-                table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
+        return PTableImpl.builderWithColumns(table, projectedColumns)
+                .setType(PTableType.PROJECTED)
+                .setBaseColumnCount(BASE_TABLE_BASE_COLUMN_COUNT)
+                .setExcludedColumns(ImmutableList.<PColumn>of())
+                .setPhysicalNames(ImmutableList.<PName>of())
+                .setColumns(projectedColumns)
+                .build();
     }
     
     public static PTable createProjectedTable(TableRef tableRef, List<ColumnRef> sourceColumnRefs, boolean retainPKColumns) throws SQLException {
@@ -192,14 +193,39 @@ public class TupleProjectionCompiler {
         if (EncodedColumnsUtil.usesEncodedColumnNames(table)) {
             cqCounter = EncodedCQCounter.copy(table.getEncodedCQCounter());
         }
-        
-        return PTableImpl.makePTable(table.getTenantId(), PROJECTED_TABLE_SCHEMA, table.getName(), PTableType.PROJECTED,
-                null, table.getTimeStamp(), table.getSequenceNumber(), table.getPKName(),
-                table.getBucketNum(), projectedColumns, null, null,
-                Collections.<PTable> emptyList(), table.isImmutableRows(), Collections.<PName> emptyList(), null, null,
-                table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(),
-                table.getViewIndexType(), table.getViewIndexId(), null, table.rowKeyOrderOptimizable(), table.getTransactionProvider(),
-                table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), cqCounter, table.useStatsForParallelization());
+        return new PTableImpl.Builder()
+                .setType(PTableType.PROJECTED)
+                .setTimeStamp(table.getTimeStamp())
+                .setIndexDisableTimestamp(table.getIndexDisableTimestamp())
+                .setSequenceNumber(table.getSequenceNumber())
+                .setImmutableRows(table.isImmutableRows())
+                .setDisableWAL(table.isWALDisabled())
+                .setMultiTenant(table.isMultiTenant())
+                .setStoreNulls(table.getStoreNulls())
+                .setViewType(table.getViewType())
+                .setViewIndexType(table.getViewIndexType())
+                .setViewIndexId(table.getViewIndexId())
+                .setTransactionProvider(table.getTransactionProvider())
+                .setUpdateCacheFrequency(table.getUpdateCacheFrequency())
+                .setNamespaceMapped(table.isNamespaceMapped())
+                .setAutoPartitionSeqName(table.getAutoPartitionSeqName())
+                .setAppendOnlySchema(table.isAppendOnlySchema())
+                .setImmutableStorageScheme(table.getImmutableStorageScheme())
+                .setQualifierEncodingScheme(table.getEncodingScheme())
+                .setBaseColumnCount(BASE_TABLE_BASE_COLUMN_COUNT)
+                .setEncodedCQCounter(cqCounter)
+                .setUseStatsForParallelization(table.useStatsForParallelization())
+                .setExcludedColumns(ImmutableList.<PColumn>of())
+                .setTenantId(table.getTenantId())
+                .setSchemaName(PROJECTED_TABLE_SCHEMA)
+                .setTableName(table.getTableName())
+                .setPkName(table.getPKName())
+                .setRowKeyOrderOptimizable(table.rowKeyOrderOptimizable())
+                .setBucketNum(table.getBucketNum())
+                .setIndexes(Collections.<PTable>emptyList())
+                .setPhysicalNames(ImmutableList.<PName>of())
+                .setColumns(projectedColumns)
+                .build();
     }
 
     // For extracting column references from single select statement

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
index c0c63eb..a54fbdd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
@@ -17,10 +17,14 @@
  */
 package org.apache.phoenix.compile;
 
+import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
+
 import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.exception.SQLExceptionInfo;
@@ -96,15 +100,34 @@ public class UnionCompiler {
             projectedColumns.add(projectedColumn);
         }
         Long scn = statement.getConnection().getSCN();
-        PTable tempTable = PTableImpl.makePTable(statement.getConnection().getTenantId(),
-            UNION_SCHEMA_NAME, UNION_TABLE_NAME, PTableType.SUBQUERY, null,
-            HConstants.LATEST_TIMESTAMP, scn == null ? HConstants.LATEST_TIMESTAMP : scn,
-            null, null, projectedColumns, null, null, null, true, null, null, null, true,
-            true, true, null, null, null, null, false, null, 0, 0L,
-            SchemaUtil.isNamespaceMappingEnabled(PTableType.SUBQUERY,
-                statement.getConnection().getQueryServices().getProps()), null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, true);
-        TableRef tableRef = new TableRef(null, tempTable, 0, false);
-        return tableRef;
+        PTable tempTable = new PTableImpl.Builder()
+                .setType(PTableType.SUBQUERY)
+                .setTimeStamp(HConstants.LATEST_TIMESTAMP)
+                .setIndexDisableTimestamp(0L)
+                .setSequenceNumber(scn == null ? HConstants.LATEST_TIMESTAMP : scn)
+                .setImmutableRows(true)
+                .setDisableWAL(true)
+                .setMultiTenant(true)
+                .setStoreNulls(true)
+                .setUpdateCacheFrequency(0)
+                .setNamespaceMapped(SchemaUtil.isNamespaceMappingEnabled(PTableType.SUBQUERY,
+                        statement.getConnection().getQueryServices().getProps()))
+                .setAppendOnlySchema(false)
+                .setImmutableStorageScheme(ImmutableStorageScheme.ONE_CELL_PER_COLUMN)
+                .setQualifierEncodingScheme(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS)
+                .setBaseColumnCount(BASE_TABLE_BASE_COLUMN_COUNT)
+                .setEncodedCQCounter(PTable.EncodedCQCounter.NULL_COUNTER)
+                .setUseStatsForParallelization(true)
+                .setExcludedColumns(ImmutableList.<PColumn>of())
+                .setTenantId(statement.getConnection().getTenantId())
+                .setSchemaName(UNION_SCHEMA_NAME)
+                .setTableName(UNION_TABLE_NAME)
+                .setRowKeyOrderOptimizable(false)
+                .setIndexes(Collections.<PTable>emptyList())
+                .setPhysicalNames(ImmutableList.<PName>of())
+                .setColumns(projectedColumns)
+                .build();
+        return new TableRef(null, tempTable, 0, false);
     }
 
     private static void compareExperssions(int i, Expression expression,

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
index 61be561..4ed0c9a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
@@ -32,6 +32,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.client.Scan;
@@ -701,9 +702,11 @@ public class UpsertCompiler {
                     }
                     // Build table from projectedColumns
                     // Hack to add default column family to be used on server in case no value column is projected.
-                    PTable projectedTable = PTableImpl.makePTable(table, projectedColumns,
-                            PNameFactory.newName(SchemaUtil.getEmptyColumnFamily(table)));  
-                    
+                    PTable projectedTable = PTableImpl.builderWithColumns(table, projectedColumns)
+                            .setExcludedColumns(ImmutableList.<PColumn>of())
+                            .setDefaultFamilyName(PNameFactory.newName(SchemaUtil.getEmptyColumnFamily(table)))
+                            .setColumns(projectedColumns)
+                            .build();
                     
                     SelectStatement select = SelectStatement.create(SelectStatement.COUNT_ONE, upsert.getHint());
                     StatementContext statementContext = queryPlan.getContext();
@@ -856,7 +859,8 @@ public class UpsertCompiler {
                     }
                     updateExpressions.add(updateExpression);
                 }
-                PTable onDupKeyTable = PTableImpl.makePTable(table, updateColumns);
+                PTable onDupKeyTable = PTableImpl.builderWithColumns(table, updateColumns)
+                        .build();
                 onDupKeyBytesToBe = PhoenixIndexBuilder.serializeOnDupKeyUpdate(onDupKeyTable, updateExpressions);
             }
         }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 424b6d6..d899e32 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -78,6 +78,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA
 import static org.apache.phoenix.query.QueryConstants.DIVERGED_VIEW_BASE_COLUMN_COUNT;
 import static org.apache.phoenix.schema.PTableType.INDEX;
 import static org.apache.phoenix.schema.PTableType.TABLE;
+import static org.apache.phoenix.schema.PTableImpl.getColumnsToClone;
 import static org.apache.phoenix.util.SchemaUtil.getVarCharLength;
 import static org.apache.phoenix.util.SchemaUtil.getVarChars;
 
@@ -101,6 +102,7 @@ import java.util.NavigableMap;
 import java.util.Properties;
 import java.util.Set;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
@@ -215,6 +217,7 @@ import org.apache.phoenix.schema.PTableImpl;
 import org.apache.phoenix.schema.PTableKey;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.ParentTableNotFoundException;
+import org.apache.phoenix.schema.RowKeySchema;
 import org.apache.phoenix.schema.SaltingUtil;
 import org.apache.phoenix.schema.SequenceAllocation;
 import org.apache.phoenix.schema.SequenceAlreadyExistsException;
@@ -689,7 +692,9 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                     }
                     indexes.add(latestIndex);
                 }
-                table = PTableImpl.makePTable(table, table.getTimeStamp(), indexes);
+                table = PTableImpl.builderWithColumns(table, getColumnsToClone(table))
+                        .setIndexes(indexes == null ? Collections.<PTable>emptyList() : indexes)
+                        .build();
             }
         }
         
@@ -939,11 +944,31 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
         int baseTableColumnCount =
                 isDiverged ? QueryConstants.DIVERGED_VIEW_BASE_COLUMN_COUNT
                         : columnsToAdd.size() - myColumns.size() + (isSalted ? 1 : 0);
+
+        // When creating a PTable for views or view indexes, use the baseTable PTable for attributes
+        // inherited from the physical base table.
+        // if a TableProperty is not valid on a view we set it to the base table value
+        // if a TableProperty is valid on a view and is not mutable on a view we set it to the base table value
+        // if a TableProperty is valid on a view and is mutable on a view we use the value set on the view
         // TODO Implement PHOENIX-4763 to set the view properties correctly instead of just
         // setting them same as the base table
-        PTableImpl pTable =
-                PTableImpl.makePTable(table, baseTable, columnsToAdd, maxTableTimestamp,
-                    baseTableColumnCount, excludedColumns);
+        PTableImpl pTable = PTableImpl.builderWithColumns(table, columnsToAdd)
+                .setImmutableRows(baseTable.isImmutableRows())
+                .setDisableWAL(baseTable.isWALDisabled())
+                .setMultiTenant(baseTable.isMultiTenant())
+                .setStoreNulls(baseTable.getStoreNulls())
+                .setTransactionProvider(baseTable.getTransactionProvider())
+                .setAutoPartitionSeqName(baseTable.getAutoPartitionSeqName())
+                .setAppendOnlySchema(baseTable.isAppendOnlySchema())
+                .setImmutableStorageScheme(baseTable.getImmutableStorageScheme() == null ?
+                        ImmutableStorageScheme.ONE_CELL_PER_COLUMN : baseTable.getImmutableStorageScheme())
+                .setQualifierEncodingScheme(baseTable.getEncodingScheme() == null ?
+                        QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : baseTable.getEncodingScheme())
+                .setBaseColumnCount(baseTableColumnCount)
+                .setTimeStamp(maxTableTimestamp)
+                .setExcludedColumns(excludedColumns == null ?
+                        ImmutableList.<PColumn>of() : ImmutableList.copyOf(excludedColumns))
+                .build();
         return WhereConstantParser.addViewInfoToPColumnsIfNeeded(pTable);
     }
 
@@ -1486,11 +1511,48 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
         }
         // Avoid querying the stats table because we're holding the rowLock here. Issuing an RPC to a remote
         // server while holding this lock is a bad idea and likely to cause contention.
-        return PTableImpl.makePTable(tenantId, schemaName, tableName, tableType, indexState, timeStamp, tableSeqNum,
-                pkName, saltBucketNum, columns, parentSchemaName, parentTableName, indexes, isImmutableRows, physicalTables, defaultFamilyName,
-                viewStatement, disableWAL, multiTenant, storeNulls, viewType, viewIndexType, viewIndexId, indexType,
-                rowKeyOrderOptimizable, transactionProvider, updateCacheFrequency, baseColumnCount,
-                indexDisableTimestamp, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, storageScheme, encodingScheme, cqCounter, useStatsForParallelization);
+        return new PTableImpl.Builder()
+                .setType(tableType)
+                .setState(indexState)
+                .setTimeStamp(timeStamp)
+                .setIndexDisableTimestamp(indexDisableTimestamp)
+                .setSequenceNumber(tableSeqNum)
+                .setImmutableRows(isImmutableRows)
+                .setViewStatement(viewStatement)
+                .setDisableWAL(disableWAL)
+                .setMultiTenant(multiTenant)
+                .setStoreNulls(storeNulls)
+                .setViewType(viewType)
+                .setViewIndexType(viewIndexType)
+                .setViewIndexId(viewIndexId)
+                .setIndexType(indexType)
+                .setTransactionProvider(transactionProvider)
+                .setUpdateCacheFrequency(updateCacheFrequency)
+                .setNamespaceMapped(isNamespaceMapped)
+                .setAutoPartitionSeqName(autoPartitionSeq)
+                .setAppendOnlySchema(isAppendOnlySchema)
+                .setImmutableStorageScheme(storageScheme == null ?
+                        ImmutableStorageScheme.ONE_CELL_PER_COLUMN : storageScheme)
+                .setQualifierEncodingScheme(encodingScheme == null ?
+                        QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : encodingScheme)
+                .setBaseColumnCount(baseColumnCount)
+                .setEncodedCQCounter(cqCounter)
+                .setUseStatsForParallelization(useStatsForParallelization)
+                .setExcludedColumns(ImmutableList.<PColumn>of())
+                .setTenantId(tenantId)
+                .setSchemaName(schemaName)
+                .setTableName(tableName)
+                .setPkName(pkName)
+                .setDefaultFamilyName(defaultFamilyName)
+                .setRowKeyOrderOptimizable(rowKeyOrderOptimizable)
+                .setBucketNum(saltBucketNum)
+                .setIndexes(indexes == null ? Collections.<PTable>emptyList() : indexes)
+                .setParentSchemaName(parentSchemaName)
+                .setParentTableName(parentTableName)
+                .setPhysicalNames(physicalTables == null ?
+                        ImmutableList.<PName>of() : ImmutableList.copyOf(physicalTables))
+                .setColumns(columns)
+                .build();
     }
     private Long getViewIndexId(Cell[] tableKeyValues, PDataType viewIndexType) {
         Cell viewIndexIdKv = tableKeyValues[VIEW_INDEX_ID_INDEX];
@@ -1740,7 +1802,21 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
     }
 
     private static PTable newDeletedTableMarker(long timestamp) {
-        return new PTableImpl(timestamp);
+        try {
+            return new PTableImpl.Builder()
+                    .setType(PTableType.TABLE)
+                    .setTimeStamp(timestamp)
+                    .setPkColumns(Collections.<PColumn>emptyList())
+                    .setAllColumns(Collections.<PColumn>emptyList())
+                    .setFamilyAttributes(Collections.<PColumnFamily>emptyList())
+                    .setRowKeySchema(RowKeySchema.EMPTY_SCHEMA)
+                    .setIndexes(Collections.<PTable>emptyList())
+                    .setPhysicalNames(Collections.<PName>emptyList())
+                    .build();
+        } catch (SQLException e) {
+            // Should never happen
+            return null;
+        }
     }
 
     private static PFunction newDeletedFunctionMarker(long timestamp) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index a667316..73386a2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -23,6 +23,7 @@ import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY;
 import static org.apache.phoenix.query.QueryConstants.UNGROUPED_AGG_ROW_KEY;
 import static org.apache.phoenix.query.QueryServices.MUTATE_BATCH_SIZE_ATTRIB;
 import static org.apache.phoenix.query.QueryServices.MUTATE_BATCH_SIZE_BYTES_ATTRIB;
+import static org.apache.phoenix.schema.PTableImpl.getColumnsToClone;
 import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.COMPACTION_UPDATE_STATS_ROW_COUNT;
 import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.CONCURRENT_UPDATE_STATS_ROW_COUNT;
 
@@ -411,7 +412,10 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
             logger.debug("Upgrading row key for " + region.getRegionInfo().getTable().getNameAsString());
             projectedTable = deserializeTable(descRowKeyTableBytes);
             try {
-                writeToTable = PTableImpl.makePTable(projectedTable, true);
+                writeToTable = PTableImpl.builderWithColumns(projectedTable,
+                        getColumnsToClone(projectedTable))
+                        .setRowKeyOrderOptimizable(true)
+                        .build();
             } catch (SQLException e) {
                 ServerUtil.throwIOException("Upgrade failed", e); // Impossible
             }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/WhereConstantParser.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/WhereConstantParser.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/WhereConstantParser.java
index 4528f55..9fc030c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/WhereConstantParser.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/WhereConstantParser.java
@@ -94,7 +94,8 @@ public class WhereConstantParser {
                 result.add(column);
             }
         }
-        return PTableImpl.makePTable(view, result);
+        return PTableImpl.builderWithColumns(view, result)
+                .build();
     }
 
     private static PhoenixConnection getConnectionlessConnection() throws SQLException {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 655de0d..d7e7d6d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -18,6 +18,7 @@
 package org.apache.phoenix.query;
 
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE_BYTES;
+import static org.apache.phoenix.schema.PTableImpl.getColumnsToClone;
 
 import java.sql.SQLException;
 import java.util.ArrayList;
@@ -316,7 +317,8 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
     public MetaDataMutationResult addColumn(List<Mutation> tableMetaData, PTable table, Map<String, List<Pair<String,Object>>> properties, Set<String> colFamiliesForPColumnsToBeAdded, List<PColumn> columnsToBeAdded) throws SQLException {
         List<PColumn> columns = Lists.newArrayList(table.getColumns());
         columns.addAll(columnsToBeAdded);
-        return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, 0, PTableImpl.makePTable(table, columns));
+        return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, 0,
+                PTableImpl.builderWithColumns(table, columns).build());
     }
 
     @Override
@@ -453,7 +455,10 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
         String indexTableName = SchemaUtil.getTableName(schemaName, indexName);
         PName tenantId = tenantIdBytes.length == 0 ? null : PNameFactory.newName(tenantIdBytes);
         PTable index = metaData.getTableRef(new PTableKey(tenantId, indexTableName)).getTable();
-        index = PTableImpl.makePTable(index,newState == PIndexState.USABLE ? PIndexState.ACTIVE : newState == PIndexState.UNUSABLE ? PIndexState.INACTIVE : newState);
+        index = PTableImpl.builderWithColumns(index, getColumnsToClone(index))
+                .setState(newState == PIndexState.USABLE ? PIndexState.ACTIVE :
+                        newState == PIndexState.UNUSABLE ? PIndexState.INACTIVE : newState)
+                .build();
         return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, 0, index);
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index df0e5d4..aae9fc4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -104,6 +104,7 @@ import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme.ONE_CELL_P
 import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS;
 import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
 import static org.apache.phoenix.schema.PTable.ViewType.MAPPED;
+import static org.apache.phoenix.schema.PTableImpl.getColumnsToClone;
 import static org.apache.phoenix.schema.PTableType.TABLE;
 import static org.apache.phoenix.schema.PTableType.VIEW;
 import static org.apache.phoenix.schema.types.PDataType.FALSE_BYTES;
@@ -132,6 +133,7 @@ import java.util.Properties;
 import java.util.Set;
 
 import org.apache.hadoop.hbase.HColumnDescriptor;
+import com.google.common.base.Objects;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.client.ClusterConnection;
@@ -247,6 +249,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -931,10 +934,21 @@ public class MetaDataClient {
                     + QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR + index.getName().getString());
                 // add the index table with a new name so that it does not conflict with the existing index table
                 // and set update cache frequency to that of the view
-                indexesToAdd.add(PTableImpl.makePTable(index, modifiedIndexName, viewStatement, view.getUpdateCacheFrequency(), view.getTenantId()));
+                if (Objects.equal(viewStatement, index.getViewStatement())) {
+                    indexesToAdd.add(index);
+                } else {
+                    indexesToAdd.add(PTableImpl.builderWithColumns(index, getColumnsToClone(index))
+                            .setTableName(modifiedIndexName)
+                            .setViewStatement(viewStatement)
+                            .setUpdateCacheFrequency(view.getUpdateCacheFrequency())
+                            .setTenantId(view.getTenantId())
+                            .build());
+                }
             }
         }
-        PTable allIndexesTable = PTableImpl.makePTable(view, view.getTimeStamp(), indexesToAdd);
+        PTable allIndexesTable = PTableImpl.builderWithColumns(view, getColumnsToClone(view))
+                .setIndexes(indexesToAdd == null ? Collections.<PTable>emptyList() : indexesToAdd)
+                .build();
         result.setTable(allIndexesTable);
         return true;
     }
@@ -2650,13 +2664,38 @@ public class MetaDataClient {
                 // TODO: what about stats for system catalog?
                 PName newSchemaName = PNameFactory.newName(schemaName);
                 // Column names and qualifiers and hardcoded for system tables.
-                PTable table = PTableImpl.makePTable(tenantId,newSchemaName, PNameFactory.newName(tableName), tableType,
-                        null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM,
-                        PNameFactory.newName(QueryConstants.SYSTEM_TABLE_PK_NAME), null, columns.values(), null, null,
-                        Collections.<PTable>emptyList(), isImmutableRows,
-                        Collections.<PName>emptyList(), defaultFamilyName == null ? null :
-                                PNameFactory.newName(defaultFamilyName), null,
-                        Boolean.TRUE.equals(disableWAL), false, false, null, viewIndexType, null, indexType, true, null, 0, 0L, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, ONE_CELL_PER_COLUMN, NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, true);
+                PTable table = new PTableImpl.Builder()
+                        .setType(tableType)
+                        .setTimeStamp(MetaDataProtocol.MIN_TABLE_TIMESTAMP)
+                        .setIndexDisableTimestamp(0L)
+                        .setSequenceNumber(PTable.INITIAL_SEQ_NUM)
+                        .setImmutableRows(isImmutableRows)
+                        .setDisableWAL(Boolean.TRUE.equals(disableWAL))
+                        .setMultiTenant(false)
+                        .setStoreNulls(false)
+                        .setViewIndexType(viewIndexType)
+                        .setIndexType(indexType)
+                        .setUpdateCacheFrequency(0)
+                        .setNamespaceMapped(isNamespaceMapped)
+                        .setAutoPartitionSeqName(autoPartitionSeq)
+                        .setAppendOnlySchema(isAppendOnlySchema)
+                        .setImmutableStorageScheme(ONE_CELL_PER_COLUMN)
+                        .setQualifierEncodingScheme(NON_ENCODED_QUALIFIERS)
+                        .setBaseColumnCount(QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT)
+                        .setEncodedCQCounter(PTable.EncodedCQCounter.NULL_COUNTER)
+                        .setUseStatsForParallelization(true)
+                        .setExcludedColumns(ImmutableList.<PColumn>of())
+                        .setTenantId(tenantId)
+                        .setSchemaName(newSchemaName)
+                        .setTableName(PNameFactory.newName(tableName))
+                        .setPkName(PNameFactory.newName(QueryConstants.SYSTEM_TABLE_PK_NAME))
+                        .setDefaultFamilyName(defaultFamilyName == null ? null :
+                                PNameFactory.newName(defaultFamilyName))
+                        .setRowKeyOrderOptimizable(true)
+                        .setIndexes(Collections.<PTable>emptyList())
+                        .setPhysicalNames(ImmutableList.<PName>of())
+                        .setColumns(columns.values())
+                        .build();
                 connection.addTable(table, MetaDataProtocol.MIN_TABLE_TIMESTAMP);
             }
             
@@ -2930,12 +2969,49 @@ public class MetaDataClient {
                  * for extra safety.
                  */
                 EncodedCQCounter cqCounterToBe = tableType == PTableType.VIEW ? NULL_COUNTER : cqCounter;
-                PTable table =  PTableImpl.makePTable(
-                        tenantId, newSchemaName, PNameFactory.newName(tableName), tableType, indexState, timestamp!=null ? timestamp : result.getMutationTime(),
-                        PTable.INITIAL_SEQ_NUM, pkName == null ? null : PNameFactory.newName(pkName), saltBucketNum, columns.values(),
-                        parent == null ? null : parent.getSchemaName(), parent == null ? null : parent.getTableName(), Collections.<PTable>emptyList(), isImmutableRows,
-                        physicalNames, defaultFamilyName == null ? null : PNameFactory.newName(defaultFamilyName), viewStatement, Boolean.TRUE.equals(disableWAL), multiTenant, storeNulls, viewType,
-                        viewIndexType, result.getViewIndexId(), indexType, rowKeyOrderOptimizable, transactionProvider, updateCacheFrequency, 0L, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, immutableStorageScheme, encodingScheme, cqCounterToBe, useStatsForParallelizationProp);
+                PTable table = new PTableImpl.Builder()
+                        .setType(tableType)
+                        .setState(indexState)
+                        .setTimeStamp(timestamp != null ? timestamp : result.getMutationTime())
+                        .setIndexDisableTimestamp(0L)
+                        .setSequenceNumber(PTable.INITIAL_SEQ_NUM)
+                        .setImmutableRows(isImmutableRows)
+                        .setViewStatement(viewStatement)
+                        .setDisableWAL(Boolean.TRUE.equals(disableWAL))
+                        .setMultiTenant(multiTenant)
+                        .setStoreNulls(storeNulls)
+                        .setViewType(viewType)
+                        .setViewIndexType(viewIndexType)
+                        .setViewIndexId(result.getViewIndexId())
+                        .setIndexType(indexType)
+                        .setTransactionProvider(transactionProvider)
+                        .setUpdateCacheFrequency(updateCacheFrequency)
+                        .setNamespaceMapped(isNamespaceMapped)
+                        .setAutoPartitionSeqName(autoPartitionSeq)
+                        .setAppendOnlySchema(isAppendOnlySchema)
+                        .setImmutableStorageScheme(immutableStorageScheme == null ?
+                                ImmutableStorageScheme.ONE_CELL_PER_COLUMN : immutableStorageScheme)
+                        .setQualifierEncodingScheme(encodingScheme == null ?
+                                QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : encodingScheme)
+                        .setBaseColumnCount(QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT)
+                        .setEncodedCQCounter(cqCounterToBe)
+                        .setUseStatsForParallelization(useStatsForParallelizationProp)
+                        .setExcludedColumns(ImmutableList.<PColumn>of())
+                        .setTenantId(tenantId)
+                        .setSchemaName(newSchemaName)
+                        .setTableName(PNameFactory.newName(tableName))
+                        .setPkName(pkName == null ? null : PNameFactory.newName(pkName))
+                        .setDefaultFamilyName(defaultFamilyName == null ?
+                                null : PNameFactory.newName(defaultFamilyName))
+                        .setRowKeyOrderOptimizable(rowKeyOrderOptimizable)
+                        .setBucketNum(saltBucketNum)
+                        .setIndexes(Collections.<PTable>emptyList())
+                        .setParentSchemaName((parent == null) ? null : parent.getSchemaName())
+                        .setParentTableName((parent == null) ? null : parent.getTableName())
+                        .setPhysicalNames(physicalNames == null ?
+                                ImmutableList.<PName>of() : ImmutableList.copyOf(physicalNames))
+                        .setColumns(columns.values())
+                        .build();
                 result = new MetaDataMutationResult(code, result.getMutationTime(), table, true);
                 addTableToCache(result);
                 return table;
@@ -3131,10 +3207,29 @@ public class MetaDataClient {
                                 && (table.isMultiTenant() || hasViewIndexTable)) {
                             if (hasViewIndexTable) {
                                 byte[] viewIndexPhysicalName = MetaDataUtil.getViewIndexPhysicalName(table.getPhysicalName().getBytes());
-                                PTable viewIndexTable = new PTableImpl(null,
-                                        SchemaUtil.getSchemaNameFromFullName(viewIndexPhysicalName),
-                                        SchemaUtil.getTableNameFromFullName(viewIndexPhysicalName), ts,
-                                        table.getColumnFamilies(),table.isNamespaceMapped(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.useStatsForParallelization());
+                                String viewIndexSchemaName = SchemaUtil.getSchemaNameFromFullName(viewIndexPhysicalName);
+                                String viewIndexTableName = SchemaUtil.getTableNameFromFullName(viewIndexPhysicalName);
+                                PName viewIndexName = PNameFactory.newName(SchemaUtil.getTableName(viewIndexSchemaName, viewIndexTableName));
+
+                                PTable viewIndexTable = new PTableImpl.Builder()
+                                        .setName(viewIndexName)
+                                        .setKey(new PTableKey(tenantId, viewIndexName.getString()))
+                                        .setSchemaName(PNameFactory.newName(viewIndexSchemaName))
+                                        .setTableName(PNameFactory.newName(viewIndexTableName))
+                                        .setType(PTableType.VIEW)
+                                        .setViewType(ViewType.MAPPED)
+                                        .setTimeStamp(ts)
+                                        .setPkColumns(Collections.<PColumn>emptyList())
+                                        .setAllColumns(Collections.<PColumn>emptyList())
+                                        .setRowKeySchema(RowKeySchema.EMPTY_SCHEMA)
+                                        .setIndexes(Collections.<PTable>emptyList())
+                                        .setFamilyAttributes(table.getColumnFamilies())
+                                        .setPhysicalNames(Collections.<PName>emptyList())
+                                        .setNamespaceMapped(table.isNamespaceMapped())
+                                        .setImmutableStorageScheme(table.getImmutableStorageScheme())
+                                        .setQualifierEncodingScheme(table.getEncodingScheme())
+                                        .setUseStatsForParallelization(table.useStatsForParallelization())
+                                        .build();
                                 tableRefs.add(new TableRef(null, viewIndexTable, ts, false));
                             }
                         }
@@ -3754,13 +3849,32 @@ public class MetaDataClient {
                             long ts = (scn == null ? result.getMutationTime() : scn);
                             byte[] viewIndexPhysicalName = MetaDataUtil
                                     .getViewIndexPhysicalName(table.getPhysicalName().getBytes());
-                            PTable viewIndexTable = new PTableImpl(null,
-                                    SchemaUtil.getSchemaNameFromFullName(viewIndexPhysicalName),
-                                    SchemaUtil.getTableNameFromFullName(viewIndexPhysicalName), ts,
-                                    table.getColumnFamilies(), table.isNamespaceMapped(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.useStatsForParallelization());
+                            String viewIndexSchemaName = SchemaUtil.getSchemaNameFromFullName(viewIndexPhysicalName);
+                            String viewIndexTableName = SchemaUtil.getTableNameFromFullName(viewIndexPhysicalName);
+                            PName viewIndexName = PNameFactory.newName(SchemaUtil.getTableName(viewIndexSchemaName, viewIndexTableName));
+
+                            PTable viewIndexTable = new PTableImpl.Builder()
+                                    .setName(viewIndexName)
+                                    .setKey(new PTableKey(tenantId, viewIndexName.getString()))
+                                    .setSchemaName(PNameFactory.newName(viewIndexSchemaName))
+                                    .setTableName(PNameFactory.newName(viewIndexTableName))
+                                    .setType(PTableType.VIEW)
+                                    .setViewType(ViewType.MAPPED)
+                                    .setTimeStamp(ts)
+                                    .setPkColumns(Collections.<PColumn>emptyList())
+                                    .setAllColumns(Collections.<PColumn>emptyList())
+                                    .setRowKeySchema(RowKeySchema.EMPTY_SCHEMA)
+                                    .setIndexes(Collections.<PTable>emptyList())
+                                    .setFamilyAttributes(table.getColumnFamilies())
+                                    .setPhysicalNames(Collections.<PName>emptyList())
+                                    .setNamespaceMapped(table.isNamespaceMapped())
+                                    .setImmutableStorageScheme(table.getImmutableStorageScheme())
+                                    .setQualifierEncodingScheme(table.getEncodingScheme())
+                                    .setUseStatsForParallelization(table.useStatsForParallelization())
+                                    .build();
                             List<TableRef> tableRefs = Collections.singletonList(new TableRef(null, viewIndexTable, ts, false));
                             MutationPlan plan = new PostDDLCompiler(connection).compile(tableRefs, null, null,
-                                    Collections.<PColumn> emptyList(), ts);
+                                    Collections.<PColumn>emptyList(), ts);
                             connection.getQueryServices().updateData(plan);
                         }
                     }
@@ -4029,21 +4143,44 @@ public class MetaDataClient {
                         Map<String, List<TableRef>> tenantIdTableRefMap = Maps.newHashMap();
                         if (result.getSharedTablesToDelete() != null) {
                             for (SharedTableState sharedTableState : result.getSharedTablesToDelete()) {
-                                PTableImpl viewIndexTable =
-                                        new PTableImpl(sharedTableState.getTenantId(),
-                                                sharedTableState.getSchemaName(),
-                                                sharedTableState.getTableName(), ts,
-                                                table.getColumnFamilies(),
-                                                sharedTableState.getColumns(),
-                                                sharedTableState.getPhysicalNames(),
-                                                sharedTableState.getViewIndexType(),
-                                                sharedTableState.getViewIndexId(),
-                                                table.isMultiTenant(), table.isNamespaceMapped(),
-                                                table.getImmutableStorageScheme(),
-                                                table.getEncodingScheme(),
-                                                table.getEncodedCQCounter(),
-                                                table.useStatsForParallelization(),
-                                                table.getBucketNum());
+                                ImmutableStorageScheme storageScheme = table.getImmutableStorageScheme();
+                                QualifierEncodingScheme qualifierEncodingScheme = table.getEncodingScheme();
+                                List<PColumn> columns = sharedTableState.getColumns();
+                                if (table.getBucketNum() != null) {
+                                    columns = columns.subList(1, columns.size());
+                                }
+
+                                PTableImpl viewIndexTable = new PTableImpl.Builder()
+                                        .setPkColumns(Collections.<PColumn>emptyList())
+                                        .setAllColumns(Collections.<PColumn>emptyList())
+                                        .setRowKeySchema(RowKeySchema.EMPTY_SCHEMA)
+                                        .setIndexes(Collections.<PTable>emptyList())
+                                        .setFamilyAttributes(table.getColumnFamilies())
+                                        .setType(PTableType.INDEX)
+                                        .setTimeStamp(ts)
+                                        .setMultiTenant(table.isMultiTenant())
+                                        .setViewIndexType(sharedTableState.getViewIndexType())
+                                        .setViewIndexId(sharedTableState.getViewIndexId())
+                                        .setNamespaceMapped(table.isNamespaceMapped())
+                                        .setAppendOnlySchema(false)
+                                        .setImmutableStorageScheme(storageScheme == null ?
+                                                ImmutableStorageScheme.ONE_CELL_PER_COLUMN : storageScheme)
+                                        .setQualifierEncodingScheme(qualifierEncodingScheme == null ?
+                                                QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : qualifierEncodingScheme)
+                                        .setEncodedCQCounter(table.getEncodedCQCounter())
+                                        .setUseStatsForParallelization(table.useStatsForParallelization())
+                                        .setExcludedColumns(ImmutableList.<PColumn>of())
+                                        .setTenantId(sharedTableState.getTenantId())
+                                        .setSchemaName(sharedTableState.getSchemaName())
+                                        .setTableName(sharedTableState.getTableName())
+                                        .setRowKeyOrderOptimizable(false)
+                                        .setBucketNum(table.getBucketNum())
+                                        .setIndexes(Collections.<PTable>emptyList())
+                                        .setPhysicalNames(sharedTableState.getPhysicalNames() == null ?
+                                                ImmutableList.<PName>of() :
+                                                ImmutableList.copyOf(sharedTableState.getPhysicalNames()))
+                                        .setColumns(columns)
+                                        .build();
                                 TableRef indexTableRef = new TableRef(viewIndexTable);
                                 PName indexTableTenantId = sharedTableState.getTenantId();
                                 if (indexTableTenantId==null) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
index 598cc79..3ce70ea 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
@@ -17,7 +17,10 @@
  */
 package org.apache.phoenix.schema;
 
+import static org.apache.phoenix.schema.PTableImpl.getColumnsToClone;
+
 import java.sql.SQLException;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
@@ -27,7 +30,6 @@ import org.apache.phoenix.parse.PSchema;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.query.QueryServicesOptions;
 import org.apache.phoenix.util.ReadOnlyProps;
-import org.apache.phoenix.util.SchemaUtil;
 import org.apache.phoenix.util.TimeKeeper;
 
 import com.google.common.collect.Lists;
@@ -121,7 +123,11 @@ public class PMetaDataImpl implements PMetaData {
                 }
                 newIndexes.add(table);
                 netGain -= oldParentRef.getEstimatedSize();
-                newParentTable = PTableImpl.makePTable(oldParentRef.getTable(), table.getTimeStamp(), newIndexes);
+                newParentTable = PTableImpl.builderWithColumns(oldParentRef.getTable(),
+                        getColumnsToClone(oldParentRef.getTable()))
+                        .setIndexes(newIndexes)
+                        .setTimeStamp(table.getTimeStamp())
+                        .build();
                 newParentTableRef = tableRefFactory.makePTableRef(newParentTable, this.timeKeeper.getCurrentTime(), parentResolvedTimestamp);
                 netGain += newParentTableRef.getEstimatedSize();
             }
@@ -173,10 +179,14 @@ public class PMetaDataImpl implements PMetaData {
                     PTable index = newIndexes.get(i);
                     if (index.getName().getString().equals(tableName)) {
                         newIndexes.remove(i);
-                        PTable parentTable = PTableImpl.makePTable(
-                                parentTableRef.getTable(),
-                                tableTimeStamp == HConstants.LATEST_TIMESTAMP ? parentTableRef.getTable().getTimeStamp() : tableTimeStamp,
-                                newIndexes);
+                        PTableImpl.Builder parentTableBuilder =
+                                PTableImpl.builderWithColumns(parentTableRef.getTable(),
+                                        getColumnsToClone(parentTableRef.getTable()))
+                                .setIndexes(newIndexes == null ? Collections.<PTable>emptyList() : newIndexes);
+                        if (tableTimeStamp != HConstants.LATEST_TIMESTAMP) {
+                            parentTableBuilder.setTimeStamp(tableTimeStamp);
+                        }
+                        PTable parentTable = parentTableBuilder.build();
                         metaData.put(parentTable.getKey(), tableRefFactory.makePTableRef(parentTable, this.timeKeeper.getCurrentTime(), parentTableRef.getResolvedTimeStamp()));
                         break;
                     }
@@ -218,8 +228,10 @@ public class PMetaDataImpl implements PMetaData {
                     oldColumn.getTimestamp());
                 columns.add(newColumn);
             }
-            
-            table = PTableImpl.makePTable(table, tableTimeStamp, tableSeqNum, columns);
+            table = PTableImpl.builderWithColumns(table, columns)
+                    .setTimeStamp(tableTimeStamp)
+                    .setSequenceNumber(tableSeqNum)
+                    .build();
         }
         tables.put(table.getKey(), tableRefFactory.makePTableRef(table, this.timeKeeper.getCurrentTime(), resolvedTime));
     }


[10/22] phoenix git commit: PHOENIX-4981 Add tests for ORDER BY, GROUP BY and salted tables using phoenix-spark

Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/src/main/java/org/apache/phoenix/spark/SparkResultSet.java
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/main/java/org/apache/phoenix/spark/SparkResultSet.java b/phoenix-spark/src/main/java/org/apache/phoenix/spark/SparkResultSet.java
new file mode 100644
index 0000000..0cb8009
--- /dev/null
+++ b/phoenix-spark/src/main/java/org/apache/phoenix/spark/SparkResultSet.java
@@ -0,0 +1,1056 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.spark;
+
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.exception.SQLExceptionInfo;
+import org.apache.phoenix.util.SQLCloseable;
+import org.apache.spark.sql.Row;
+
+import java.io.InputStream;
+import java.io.Reader;
+import java.math.BigDecimal;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.sql.Array;
+import java.sql.Blob;
+import java.sql.Clob;
+import java.sql.Date;
+import java.sql.NClob;
+import java.sql.Ref;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.RowId;
+import java.sql.SQLException;
+import java.sql.SQLFeatureNotSupportedException;
+import java.sql.SQLWarning;
+import java.sql.SQLXML;
+import java.sql.Statement;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Helper class to convert a List of Rows returned from a dataset to a sql ResultSet
+ */
+public class SparkResultSet implements ResultSet, SQLCloseable {
+
+    private int index = -1;
+    private List<Row> dataSetRows;
+    private List<String> columnNames;
+    private boolean wasNull = false;
+
+    public SparkResultSet(List<Row> rows, String[] columnNames) {
+        this.dataSetRows = rows;
+        this.columnNames = Arrays.asList(columnNames);
+    }
+
+    private Row getCurrentRow() {
+        return dataSetRows.get(index);
+    }
+
+    @Override
+    public boolean absolute(int row) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void afterLast() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void beforeFirst() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void cancelRowUpdates() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void clearWarnings() throws SQLException {
+    }
+
+    @Override
+    public void close() throws SQLException {
+    }
+
+    @Override
+    public void deleteRow() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public int findColumn(String columnLabel) throws SQLException {
+        return columnNames.indexOf(columnLabel.toUpperCase())+1;
+    }
+
+    @Override
+    public boolean first() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Array getArray(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Array getArray(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public InputStream getAsciiStream(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public InputStream getAsciiStream(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    private void checkOpen() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    private void checkCursorState() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public BigDecimal getBigDecimal(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public BigDecimal getBigDecimal(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public InputStream getBinaryStream(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public InputStream getBinaryStream(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Blob getBlob(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Blob getBlob(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public boolean getBoolean(int columnIndex) throws SQLException {
+        wasNull = getCurrentRow().isNullAt(columnIndex-1);
+        return wasNull ? false : getCurrentRow().getBoolean(columnIndex-1);
+    }
+
+    @Override
+    public boolean getBoolean(String columnLabel) throws SQLException {
+        return getBoolean(findColumn(columnLabel));
+    }
+
+    @Override
+    public byte[] getBytes(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public byte[] getBytes(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public byte getByte(int columnIndex) throws SQLException {
+        wasNull = getCurrentRow().isNullAt(columnIndex-1);
+        return wasNull ? 0 : getCurrentRow().getByte(columnIndex-1);
+    }
+
+    @Override
+    public byte getByte(String columnLabel) throws SQLException {
+        return getByte(findColumn(columnLabel));
+    }
+
+    @Override
+    public Reader getCharacterStream(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Reader getCharacterStream(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Clob getClob(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Clob getClob(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public int getConcurrency() throws SQLException {
+        return ResultSet.CONCUR_READ_ONLY;
+    }
+
+    @Override
+    public String getCursorName() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Date getDate(int columnIndex) throws SQLException {
+        wasNull = getCurrentRow().isNullAt(columnIndex-1);
+        return getCurrentRow().getDate(columnIndex-1);
+    }
+
+    @Override
+    public Date getDate(String columnLabel) throws SQLException {
+        return getDate(findColumn(columnLabel));
+    }
+
+    @Override
+    public Date getDate(int columnIndex, Calendar cal) throws SQLException {
+        cal.setTime(getCurrentRow().getDate(columnIndex-1));
+        return new Date(cal.getTimeInMillis());
+    }
+
+    @Override
+    public Date getDate(String columnLabel, Calendar cal) throws SQLException {
+        return getDate(findColumn(columnLabel), cal);
+    }
+
+    @Override
+    public double getDouble(int columnIndex) throws SQLException {
+        wasNull = getCurrentRow().isNullAt(columnIndex-1);
+        return wasNull ? 0 : getCurrentRow().getDouble(columnIndex-1);
+    }
+
+    @Override
+    public double getDouble(String columnLabel) throws SQLException {
+        return getDouble(findColumn(columnLabel));
+    }
+
+    @Override
+    public int getFetchDirection() throws SQLException {
+        return ResultSet.FETCH_FORWARD;
+    }
+
+    @Override
+    public int getFetchSize() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public float getFloat(int columnIndex) throws SQLException {
+        wasNull = getCurrentRow().isNullAt(columnIndex-1);
+        return wasNull ? 0 : getCurrentRow().getFloat(columnIndex-1);
+    }
+
+    @Override
+    public float getFloat(String columnLabel) throws SQLException {
+        return getFloat(findColumn(columnLabel));
+    }
+
+    @Override
+    public int getHoldability() throws SQLException {
+        return ResultSet.CLOSE_CURSORS_AT_COMMIT;
+    }
+
+    @Override
+    public int getInt(int columnIndex) throws SQLException {
+        wasNull = getCurrentRow().isNullAt(columnIndex-1);
+        return wasNull ? 0 :  getCurrentRow().getInt(columnIndex-1);
+    }
+
+    @Override
+    public int getInt(String columnLabel) throws SQLException {
+        return getInt(findColumn(columnLabel));
+    }
+
+    @Override
+    public long getLong(int columnIndex) throws SQLException {
+        wasNull = getCurrentRow().isNullAt(columnIndex-1);
+        return wasNull ? 0 :  getCurrentRow().getLong(columnIndex-1);
+    }
+
+    @Override
+    public long getLong(String columnLabel) throws SQLException {
+        return getLong(findColumn(columnLabel));
+    }
+
+    @Override
+    public ResultSetMetaData getMetaData() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Reader getNCharacterStream(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Reader getNCharacterStream(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public NClob getNClob(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public NClob getNClob(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public String getNString(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public String getNString(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Object getObject(int columnIndex) throws SQLException {
+        return getCurrentRow().get(columnIndex-1);
+    }
+
+    @Override
+    public Object getObject(String columnLabel) throws SQLException {
+        return getObject(findColumn(columnLabel));
+    }
+
+    @Override
+    public Object getObject(int columnIndex, Map<String, Class<?>> map) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Object getObject(String columnLabel, Map<String, Class<?>> map) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Ref getRef(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Ref getRef(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public int getRow() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public RowId getRowId(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public RowId getRowId(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public SQLXML getSQLXML(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public SQLXML getSQLXML(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public short getShort(int columnIndex) throws SQLException {
+        return getCurrentRow().getShort(columnIndex-1);
+    }
+
+    @Override
+    public short getShort(String columnLabel) throws SQLException {
+        return getShort(findColumn(columnLabel));
+    }
+
+    @Override
+    public Statement getStatement() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public String getString(int columnIndex) throws SQLException {
+        wasNull = getCurrentRow().isNullAt(columnIndex-1);
+        return wasNull ? null : getCurrentRow().getString(columnIndex-1);
+    }
+
+    @Override
+    public String getString(String columnLabel) throws SQLException {
+        return getString(findColumn(columnLabel));
+    }
+
+    @Override
+    public Time getTime(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Time getTime(String columnLabel) throws SQLException {
+        return getTime(findColumn(columnLabel));
+    }
+
+    @Override
+    public Time getTime(int columnIndex, Calendar cal) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public Time getTime(String columnLabel, Calendar cal) throws SQLException {
+        return getTime(findColumn(columnLabel),cal);
+    }
+
+    @Override
+    public Timestamp getTimestamp(int columnIndex) throws SQLException {
+        return getCurrentRow().getTimestamp(columnIndex-1);
+    }
+
+    @Override
+    public Timestamp getTimestamp(String columnLabel) throws SQLException {
+        return getTimestamp(findColumn(columnLabel));
+    }
+
+    @Override
+    public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException {
+        return getTimestamp(columnIndex-1);
+    }
+
+    @Override
+    public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException {
+        return getTimestamp(findColumn(columnLabel),cal);
+    }
+
+    @Override
+    public int getType() throws SQLException {
+        return ResultSet.TYPE_FORWARD_ONLY;
+    }
+
+    @Override
+    public URL getURL(int columnIndex) throws SQLException {
+        try {
+            return new URL(getCurrentRow().getString(columnIndex-1));
+        } catch (MalformedURLException e) {
+            throw new SQLExceptionInfo.Builder(SQLExceptionCode.MALFORMED_URL).setRootCause(e)
+                    .build().buildException();
+        }
+    }
+
+    @Override
+    public URL getURL(String columnLabel) throws SQLException {
+        return getURL(findColumn(columnLabel));
+    }
+
+    @Override
+    public InputStream getUnicodeStream(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public InputStream getUnicodeStream(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public SQLWarning getWarnings() throws SQLException {
+        return null;
+    }
+
+    @Override
+    public void insertRow() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public boolean isAfterLast() throws SQLException {
+        return index >= dataSetRows.size();
+    }
+
+    @Override
+    public boolean isBeforeFirst() throws SQLException {
+        return index == -1;
+    }
+
+    @Override
+    public boolean isClosed() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public boolean isFirst() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public boolean isLast() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public boolean last() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void moveToCurrentRow() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void moveToInsertRow() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public boolean next() throws SQLException {
+        index++;
+        return index < dataSetRows.size();
+    }
+
+    @Override
+    public boolean previous() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void refreshRow() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public boolean relative(int rows) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public boolean rowDeleted() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public boolean rowInserted() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public boolean rowUpdated() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void setFetchDirection(int direction) throws SQLException {
+        if (direction != ResultSet.FETCH_FORWARD) {
+            throw new SQLFeatureNotSupportedException();
+        }
+    }
+
+    @Override
+    public void setFetchSize(int rows) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateArray(int columnIndex, Array x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateArray(String columnLabel, Array x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateAsciiStream(int columnIndex, InputStream x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateAsciiStream(String columnLabel, InputStream x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateAsciiStream(int columnIndex, InputStream x, long length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateAsciiStream(String columnLabel, InputStream x, long length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBinaryStream(int columnIndex, InputStream x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBinaryStream(String columnLabel, InputStream x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBinaryStream(String columnLabel, InputStream x, long length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBlob(int columnIndex, Blob x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBlob(String columnLabel, Blob x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBlob(int columnIndex, InputStream inputStream) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBlob(String columnLabel, InputStream inputStream) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBlob(int columnIndex, InputStream inputStream, long length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBlob(String columnLabel, InputStream inputStream, long length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBoolean(int columnIndex, boolean x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBoolean(String columnLabel, boolean x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateByte(int columnIndex, byte x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateByte(String columnLabel, byte x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBytes(int columnIndex, byte[] x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateBytes(String columnLabel, byte[] x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateCharacterStream(int columnIndex, Reader x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateCharacterStream(String columnLabel, Reader reader) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateCharacterStream(int columnIndex, Reader x, long length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateCharacterStream(String columnLabel, Reader reader, long length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateClob(int columnIndex, Clob x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateClob(String columnLabel, Clob x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateClob(int columnIndex, Reader reader) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateClob(String columnLabel, Reader reader) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateClob(int columnIndex, Reader reader, long length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateClob(String columnLabel, Reader reader, long length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateDate(int columnIndex, Date x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateDate(String columnLabel, Date x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateDouble(int columnIndex, double x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateDouble(String columnLabel, double x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateFloat(int columnIndex, float x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateFloat(String columnLabel, float x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateInt(int columnIndex, int x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateInt(String columnLabel, int x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateLong(int columnIndex, long x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateLong(String columnLabel, long x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateNCharacterStream(int columnIndex, Reader x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateNCharacterStream(String columnLabel, Reader reader) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateNCharacterStream(int columnIndex, Reader x, long length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateNCharacterStream(String columnLabel, Reader reader, long length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateNClob(int columnIndex, NClob nClob) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateNClob(String columnLabel, NClob nClob) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateNClob(int columnIndex, Reader reader) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateNClob(String columnLabel, Reader reader) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateNClob(int columnIndex, Reader reader, long length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateNClob(String columnLabel, Reader reader, long length) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateNString(int columnIndex, String nString) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateNString(String columnLabel, String nString) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateNull(int columnIndex) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateNull(String columnLabel) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateObject(int columnIndex, Object x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateObject(String columnLabel, Object x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateRef(int columnIndex, Ref x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateRef(String columnLabel, Ref x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateRow() throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateRowId(int columnIndex, RowId x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateRowId(String columnLabel, RowId x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateSQLXML(int columnIndex, SQLXML xmlObject) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateSQLXML(String columnLabel, SQLXML xmlObject) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateShort(int columnIndex, short x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateShort(String columnLabel, short x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateString(int columnIndex, String x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateString(String columnLabel, String x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateTime(int columnIndex, Time x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateTime(String columnLabel, Time x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException {
+        throw new SQLFeatureNotSupportedException();
+    }
+
+    @Override
+    public boolean wasNull() throws SQLException {
+        return wasNull;
+    }
+
+    @Override
+    public boolean isWrapperFor(Class<?> iface) throws SQLException {
+        return iface.isInstance(this);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <T> T unwrap(Class<T> iface) throws SQLException {
+        if (!iface.isInstance(this)) {
+            throw new SQLExceptionInfo.Builder(SQLExceptionCode.CLASS_NOT_UNWRAPPABLE)
+                    .setMessage(this.getClass().getName() + " not unwrappable from " + iface.getName())
+                    .build().buildException();
+        }
+        return (T)this;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <T> T getObject(int columnIndex, Class<T> type) throws SQLException {
+        return (T) getObject(columnIndex-1); // Just ignore type since we only support built-in types
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <T> T getObject(String columnLabel, Class<T> type) throws SQLException {
+        return (T) getObject(columnLabel); // Just ignore type since we only support built-in types
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala b/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala
index 2c2c6e1..d604e0e 100644
--- a/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala
+++ b/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala
@@ -21,8 +21,9 @@ import org.apache.hadoop.io.NullWritable
 import org.apache.phoenix.jdbc.PhoenixDriver
 import org.apache.phoenix.mapreduce.PhoenixInputFormat
 import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil
+import org.apache.phoenix.query.QueryConstants
 import org.apache.phoenix.schema.types._
-import org.apache.phoenix.util.ColumnInfo
+import org.apache.phoenix.util.{ColumnInfo, SchemaUtil}
 import org.apache.spark._
 import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.rdd.RDD
@@ -148,13 +149,25 @@ class PhoenixRDD(sc: SparkContext, table: String, columns: Seq[String],
     }), new StructType(structFields))
   }
 
-  def phoenixSchemaToCatalystSchema(columnList: Seq[ColumnInfo]) = {
-    columnList.map(ci => {
-      val structType = phoenixTypeToCatalystType(ci)
-      StructField(ci.getDisplayName, structType)
-    })
+  def normalizeColumnName(columnName: String) = {
+    val unescapedColumnName = SchemaUtil.getUnEscapedFullColumnName(columnName)
+    var normalizedColumnName = ""
+    if (unescapedColumnName.indexOf(QueryConstants.NAME_SEPARATOR) < 0) {
+      normalizedColumnName = unescapedColumnName
+    }
+    else {
+      // split by separator to get the column family and column name
+      val tokens = unescapedColumnName.split(QueryConstants.NAME_SEPARATOR_REGEX)
+      normalizedColumnName = if (tokens(0) == "0") tokens(1) else unescapedColumnName
+    }
+    normalizedColumnName
   }
 
+  def phoenixSchemaToCatalystSchema(columnList: Seq[ColumnInfo]) = columnList.map(ci => {
+    val structType = phoenixTypeToCatalystType(ci)
+    StructField(normalizeColumnName(ci.getColumnName), structType)
+  })
+
 
   // Lookup table for Phoenix types to Spark catalyst types
   def phoenixTypeToCatalystType(columnInfo: ColumnInfo): DataType = columnInfo.getPDataType match {
@@ -166,7 +179,7 @@ class PhoenixRDD(sc: SparkContext, table: String, columns: Seq[String],
     case t if t.isInstanceOf[PFloat] || t.isInstanceOf[PUnsignedFloat] => FloatType
     case t if t.isInstanceOf[PDouble] || t.isInstanceOf[PUnsignedDouble] => DoubleType
     // Use Spark system default precision for now (explicit to work with < 1.5)
-    case t if t.isInstanceOf[PDecimal] => 
+    case t if t.isInstanceOf[PDecimal] =>
       if (columnInfo.getPrecision == null || columnInfo.getPrecision < 0) DecimalType(38, 18) else DecimalType(columnInfo.getPrecision, columnInfo.getScale)
     case t if t.isInstanceOf[PTimestamp] || t.isInstanceOf[PUnsignedTimestamp] => TimestampType
     case t if t.isInstanceOf[PTime] || t.isInstanceOf[PUnsignedTime] => TimestampType

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 143f285..eba69a9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,7 +101,7 @@
     <avatica.version>1.12.0</avatica.version>
     <jettyVersion>8.1.7.v20120910</jettyVersion>
     <tephra.version>0.15.0-incubating</tephra.version>
-    <spark.version>2.0.2</spark.version>
+    <spark.version>2.3.2</spark.version>
     <scala.version>2.11.8</scala.version>
     <scala.binary.version>2.11</scala.binary.version>
     <stream.version>2.9.5</stream.version>


[09/22] phoenix git commit: PHOENIX-4997 Phoenix MR on snapshots can produce duplicate rows

Posted by ja...@apache.org.
PHOENIX-4997 Phoenix MR on snapshots can produce duplicate rows


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/1b2a3d5c
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/1b2a3d5c
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/1b2a3d5c

Branch: refs/heads/omid2
Commit: 1b2a3d5c7f25864158cd5062f2cd58d834fb5e1f
Parents: c3ed4f8
Author: Karan Mehta <ka...@gmail.com>
Authored: Thu Nov 1 17:15:26 2018 -0700
Committer: Karan Mehta <ka...@gmail.com>
Committed: Thu Nov 1 17:54:37 2018 -0700

----------------------------------------------------------------------
 .../end2end/TableSnapshotReadsMapReduceIT.java  | 122 +++++++++++--------
 .../iterate/MapReduceParallelScanGrouper.java   |  32 ++++-
 .../iterate/TableSnapshotResultIterator.java    |  28 +++--
 .../java/org/apache/phoenix/query/BaseTest.java |  14 +--
 4 files changed, 122 insertions(+), 74 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/1b2a3d5c/phoenix-core/src/it/java/org/apache/phoenix/end2end/TableSnapshotReadsMapReduceIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TableSnapshotReadsMapReduceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TableSnapshotReadsMapReduceIT.java
index cae91a3..e35e159 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TableSnapshotReadsMapReduceIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TableSnapshotReadsMapReduceIT.java
@@ -36,6 +36,7 @@ import java.util.UUID;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -49,12 +50,18 @@ import org.apache.phoenix.mapreduce.index.PhoenixIndexDBWritable;
 import org.apache.phoenix.mapreduce.util.PhoenixMapReduceUtil;
 import org.apache.phoenix.util.ReadOnlyProps;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TableSnapshotReadsMapReduceIT extends BaseUniqueNamesOwnClusterIT {
+
+  private static final Logger logger = LoggerFactory.getLogger(TableSnapshotReadsMapReduceIT.class);
+
   private final static String SNAPSHOT_NAME = "FOO";
   private static final String FIELD1 = "FIELD1";
   private static final String FIELD2 = "FIELD2";
@@ -66,6 +73,9 @@ public class TableSnapshotReadsMapReduceIT extends BaseUniqueNamesOwnClusterIT {
   private static List<List<Object>> result;
   private long timestamp;
   private String tableName;
+  private Job job;
+  private Path tmpDir;
+  private Configuration conf;
 
   @BeforeClass
   public static void doSetup() throws Exception {
@@ -73,8 +83,8 @@ public class TableSnapshotReadsMapReduceIT extends BaseUniqueNamesOwnClusterIT {
       setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
   }
 
-  @Test
-  public void testMapReduceSnapshots() throws Exception {
+  @Before
+  public void before() throws SQLException, IOException {
     // create table
     Connection conn = DriverManager.getConnection(getUrl());
     tableName = generateUniqueName();
@@ -82,58 +92,43 @@ public class TableSnapshotReadsMapReduceIT extends BaseUniqueNamesOwnClusterIT {
     conn.commit();
 
     // configure Phoenix M/R job to read snapshot
-    final Configuration conf = getUtility().getConfiguration();
-    Job job = Job.getInstance(conf);
-    Path tmpDir = getUtility().getRandomDir();
+    conf = getUtility().getConfiguration();
+    job = Job.getInstance(conf);
+    tmpDir = getUtility().getRandomDir();
+  }
 
-    PhoenixMapReduceUtil.setInput(job,PhoenixIndexDBWritable.class,SNAPSHOT_NAME,tableName,tmpDir, null, FIELD1, FIELD2, FIELD3);
+  @Test
+  public void testMapReduceSnapshots() throws Exception {
+    PhoenixMapReduceUtil.setInput(job,PhoenixIndexDBWritable.class,
+            SNAPSHOT_NAME, tableName, tmpDir, null, FIELD1, FIELD2, FIELD3);
+    configureJob(job, tableName, null, null, false);
+  }
 
-    // configure and test job
-    configureJob(job, tableName, null, null);
+  @Test
+  public void testMapReduceSnapshotsMultiRegion() throws Exception {
+    PhoenixMapReduceUtil.setInput(job,PhoenixIndexDBWritable.class,
+            SNAPSHOT_NAME, tableName, tmpDir, null, FIELD1, FIELD2, FIELD3);
+    configureJob(job, tableName, null, null, true);
   }
 
   @Test
   public void testMapReduceSnapshotsWithCondition() throws Exception {
-    // create table
-    Connection conn = DriverManager.getConnection(getUrl());
-    tableName = generateUniqueName();
-    conn.createStatement().execute(String.format(CREATE_TABLE, tableName));
-    conn.commit();
-
-    // configure Phoenix M/R job to read snapshot
-    final Configuration conf = getUtility().getConfiguration();
-    Job job = Job.getInstance(conf);
-    Path tmpDir = getUtility().getRandomDir();
-    PhoenixMapReduceUtil.setInput(job,PhoenixIndexDBWritable.class,SNAPSHOT_NAME,tableName,tmpDir, FIELD3 + " > 0001", FIELD1, FIELD2, FIELD3);
-
-    // configure and test job
-    configureJob(job, tableName, null, "FIELD3 > 0001");
-
+    PhoenixMapReduceUtil.setInput(job,PhoenixIndexDBWritable.class,
+            SNAPSHOT_NAME, tableName, tmpDir, FIELD3 + " > 0001", FIELD1, FIELD2, FIELD3);
+    configureJob(job, tableName, null, "FIELD3 > 0001", false);
   }
 
   @Test
   public void testMapReduceSnapshotWithLimit() throws Exception {
-    // create table
-    Connection conn = DriverManager.getConnection(getUrl());
-    tableName = generateUniqueName();
-    conn.createStatement().execute(String.format(CREATE_TABLE, tableName));
-    conn.commit();
-
-    // configure Phoenix M/R job to read snapshot
-    final Configuration conf = getUtility().getConfiguration();
-    Job job = Job.getInstance(conf);
-    Path tmpDir = getUtility().getRandomDir();
-    // Running limit with order by on non pk column
     String inputQuery = "SELECT * FROM " + tableName + " ORDER BY FIELD2 LIMIT 1";
-    PhoenixMapReduceUtil.setInput(job,PhoenixIndexDBWritable.class,SNAPSHOT_NAME,tableName,tmpDir,inputQuery);
-
-    // configure and test job
-    configureJob(job, tableName, inputQuery, null);
+    PhoenixMapReduceUtil.setInput(job,PhoenixIndexDBWritable.class,
+            SNAPSHOT_NAME, tableName, tmpDir, inputQuery);
+    configureJob(job, tableName, inputQuery, null, false);
   }
 
-  private void configureJob(Job job, String tableName, String inputQuery, String condition) throws Exception {
+  private void configureJob(Job job, String tableName, String inputQuery, String condition, boolean shouldSplit) throws Exception {
     try {
-      upsertAndSnapshot(tableName);
+      upsertAndSnapshot(tableName, shouldSplit);
       result = new ArrayList<>();
 
       job.setMapperClass(TableSnapshotMapper.class);
@@ -151,6 +146,7 @@ public class TableSnapshotReadsMapReduceIT extends BaseUniqueNamesOwnClusterIT {
       if (condition != null) {
         selectQuery.append(" WHERE " + condition);
       }
+
       if (inputQuery == null)
         inputQuery = selectQuery.toString();
 
@@ -176,12 +172,13 @@ public class TableSnapshotReadsMapReduceIT extends BaseUniqueNamesOwnClusterIT {
   private void upsertData(String tableName) throws SQLException {
     Connection conn = DriverManager.getConnection(getUrl());
     PreparedStatement stmt = conn.prepareStatement(String.format(UPSERT, tableName));
-    upsertData(stmt, "CCCC", "SSDD", 0001);
-    upsertData(stmt, "CCCC", "HDHG", 0005);
-    upsertData(stmt, "BBBB", "JSHJ", 0002);
-    upsertData(stmt, "AAAA", "JHHD", 0003);
+    upsertData(stmt, "AAAA", "JHHD", 37);
+    upsertData(stmt, "BBBB", "JSHJ", 224);
+    upsertData(stmt, "CCCC", "SSDD", 15);
+    upsertData(stmt, "PPPP", "AJDG", 53);
+    upsertData(stmt, "SSSS", "HSDG", 59);
+    upsertData(stmt, "XXXX", "HDPP", 22);
     conn.commit();
-    timestamp = System.currentTimeMillis();
   }
 
   private void upsertData(PreparedStatement stmt, String field1, String field2, int field3) throws SQLException {
@@ -191,31 +188,52 @@ public class TableSnapshotReadsMapReduceIT extends BaseUniqueNamesOwnClusterIT {
     stmt.execute();
   }
 
-  public void upsertAndSnapshot(String tableName) throws Exception {
+  private void upsertAndSnapshot(String tableName, boolean shouldSplit) throws Exception {
     upsertData(tableName);
 
+    TableName hbaseTableName = TableName.valueOf(tableName);
     Connection conn = DriverManager.getConnection(getUrl());
     HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
-    admin.snapshot(SNAPSHOT_NAME, TableName.valueOf(tableName));
-    // call flush to create new files in the region
-    admin.flush(tableName);
+
+    if (shouldSplit) {
+      splitTableSync(admin, hbaseTableName, "BBBB".getBytes(), 2);
+    }
+
+    admin.snapshot(SNAPSHOT_NAME, hbaseTableName);
 
     List<HBaseProtos.SnapshotDescription> snapshots = admin.listSnapshots();
     Assert.assertEquals(tableName, snapshots.get(0).getTable());
 
+    // Capture the snapshot timestamp to use as SCN while reading the table later
+    // Assigning the timestamp value here will make tests less flaky
+    timestamp = System.currentTimeMillis();
+
     // upsert data after snapshot
     PreparedStatement stmt = conn.prepareStatement(String.format(UPSERT, tableName));
-    upsertData(stmt, "DDDD", "SNFB", 0004);
+    upsertData(stmt, "DDDD", "SNFB", 45);
     conn.commit();
   }
 
-    public void deleteSnapshot(String tableName) throws Exception {
+  private void splitTableSync(HBaseAdmin admin, TableName hbaseTableName,
+                              byte[] splitPoint , int expectedRegions) throws IOException, InterruptedException {
+    admin.split(hbaseTableName, splitPoint);
+    for (int i = 0; i < 100; i++) {
+      List<HRegionInfo> hRegionInfoList = admin.getTableRegions(hbaseTableName);
+      if (hRegionInfoList.size() >= expectedRegions) {
+        break;
+      }
+      logger.info("Sleeping for 1000 ms while waiting for " + hbaseTableName.getNameAsString() + " to split");
+      Thread.sleep(1000);
+    }
+  }
+
+  private void deleteSnapshot(String tableName) throws Exception {
         try (Connection conn = DriverManager.getConnection(getUrl());
                 HBaseAdmin admin =
                         conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();) {
             admin.deleteSnapshot(SNAPSHOT_NAME);
         }
-    }
+  }
 
   public static class TableSnapshotMapper extends Mapper<NullWritable, PhoenixIndexDBWritable, ImmutableBytesWritable, NullWritable> {
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/1b2a3d5c/phoenix-core/src/main/java/org/apache/phoenix/iterate/MapReduceParallelScanGrouper.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/MapReduceParallelScanGrouper.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/MapReduceParallelScanGrouper.java
index 593608f..b4f81ae 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/MapReduceParallelScanGrouper.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/MapReduceParallelScanGrouper.java
@@ -18,6 +18,7 @@
 package org.apache.phoenix.iterate;
 
 import java.sql.SQLException;
+import java.util.Collections;
 import java.util.List;
 
 import com.google.common.base.Preconditions;
@@ -45,7 +46,7 @@ public class MapReduceParallelScanGrouper implements ParallelScanGrouper {
 
 	private static final MapReduceParallelScanGrouper INSTANCE = new MapReduceParallelScanGrouper();
 
-  public static MapReduceParallelScanGrouper getInstance() {
+    public static MapReduceParallelScanGrouper getInstance() {
 		return INSTANCE;
 	}
 
@@ -79,18 +80,39 @@ public class MapReduceParallelScanGrouper implements ParallelScanGrouper {
 		}
 	}
 
+	/**
+	 * Get list of region locations from SnapshotManifest
+	 * BaseResultIterators assume that regions are sorted using RegionInfo.COMPARATOR
+	 */
 	private List<HRegionLocation> getRegionLocationsFromManifest(SnapshotManifest manifest) {
 		List<SnapshotProtos.SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
 		Preconditions.checkNotNull(regionManifests);
 
-		List<HRegionLocation> regionLocations = Lists.newArrayListWithCapacity(regionManifests.size());
+		List<HRegionInfo> regionInfos = Lists.newArrayListWithCapacity(regionManifests.size());
+		List<HRegionLocation> hRegionLocations = Lists.newArrayListWithCapacity(regionManifests.size());
 
 		for (SnapshotProtos.SnapshotRegionManifest regionManifest : regionManifests) {
-			regionLocations.add(new HRegionLocation(
-					HRegionInfo.convert(regionManifest.getRegionInfo()), null));
+			HRegionInfo regionInfo = HRegionInfo.convert(regionManifest.getRegionInfo());
+			if (isValidRegion(regionInfo)) {
+				regionInfos.add(regionInfo);
+			}
+		}
+
+		Collections.sort(regionInfos);
+
+		for (HRegionInfo regionInfo : regionInfos) {
+			hRegionLocations.add(new HRegionLocation(regionInfo, null));
 		}
 
-		return regionLocations;
+		return hRegionLocations;
+	}
+
+	// Exclude offline split parent regions
+	private boolean isValidRegion(HRegionInfo hri) {
+		if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) {
+			return false;
+		}
+		return true;
 	}
 
 	private String getSnapshotName(Configuration conf) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/1b2a3d5c/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableSnapshotResultIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableSnapshotResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableSnapshotResultIterator.java
index 016d3be..c3d75f7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableSnapshotResultIterator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableSnapshotResultIterator.java
@@ -37,7 +37,6 @@ import java.io.IOException;
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 import java.util.UUID;
 
@@ -79,22 +78,31 @@ public class TableSnapshotResultIterator implements ResultIterator {
     RestoreSnapshotHelper.RestoreMetaChanges meta =
         RestoreSnapshotHelper.copySnapshotForScanner(this.configuration, this.fs,
             this.rootDir, this.restoreDir, this.snapshotName);
-    List restoredRegions = meta.getRegionsToAdd();
+    List<HRegionInfo> restoredRegions = meta.getRegionsToAdd();
     this.htd = meta.getTableDescriptor();
-    this.regions = new ArrayList(restoredRegions.size());
-    Iterator i$ = restoredRegions.iterator();
-
-    while(i$.hasNext()) {
-      HRegionInfo hri = (HRegionInfo)i$.next();
-      if(CellUtil.overlappingKeys(this.scan.getStartRow(), this.scan.getStopRow(),
-          hri.getStartKey(), hri.getEndKey())) {
-        this.regions.add(hri);
+    this.regions = new ArrayList<>(restoredRegions.size());
+
+    for (HRegionInfo restoredRegion : restoredRegions) {
+      if (isValidRegion(restoredRegion)) {
+        this.regions.add(restoredRegion);
       }
     }
 
     Collections.sort(this.regions);
   }
 
+  /**
+   * Exclude offline split parent regions and
+   * regions that don't intersect with provided scan
+   */
+  private boolean isValidRegion(HRegionInfo hri) {
+    if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) {
+      return false;
+    }
+    return CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(),
+            hri.getStartKey(), hri.getEndKey());
+  }
+
   public boolean initSnapshotScanner() throws SQLException {
     if (closed) {
       return true;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/1b2a3d5c/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index e6e3936..42fa085 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -1782,15 +1782,15 @@ public abstract class BaseTest {
     
 
     /**
-     *  Split SYSTEM.CATALOG at the given split point 
+     *  Synchronously split table at the given split point
      */
-    protected static void splitRegion(byte[] splitPoint) throws SQLException, IOException, InterruptedException {
-        HBaseAdmin admin =
+    protected static void splitRegion(TableName fullTableName, byte[] splitPoint) throws SQLException, IOException, InterruptedException {
+         HBaseAdmin admin =
                 driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin();
-        admin.split(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME, splitPoint);
+        admin.split(fullTableName, splitPoint);
         // make sure the split finishes (there's no synchronous splitting before HBase 2.x)
-        admin.disableTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME);
-        admin.enableTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME);
+        admin.disableTable(fullTableName);
+        admin.enableTable(fullTableName);
     }
     
     /**
@@ -1819,7 +1819,7 @@ public abstract class BaseTest {
         AssignmentManager am = master.getAssignmentManager();
         // No need to split on the first splitPoint since the end key of region boundaries are exclusive
         for (int i=1; i<splitPoints.size(); ++i) {
-            splitRegion(splitPoints.get(i));
+            splitRegion(fullTableName, splitPoints.get(i));
         }
         HashMap<ServerName, List<HRegionInfo>> serverToRegionsList = Maps.newHashMapWithExpectedSize(NUM_SLAVES_BASE);
         Deque<ServerName> availableRegionServers = new ArrayDeque<ServerName>(NUM_SLAVES_BASE);


[02/22] phoenix git commit: PHOENIX-4764 Cleanup metadata of child views for a base table that has been dropped

Posted by ja...@apache.org.
PHOENIX-4764 Cleanup metadata of child views for a base table that has been dropped


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3903ad76
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3903ad76
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3903ad76

Branch: refs/heads/omid2
Commit: 3903ad768496d0a6517298a8f196863b87290e72
Parents: a48e42e
Author: Kadir <ko...@salesforce.com>
Authored: Wed Sep 26 23:32:31 2018 -0700
Committer: Karan Mehta <ka...@gmail.com>
Committed: Fri Oct 26 14:40:48 2018 -0700

----------------------------------------------------------------------
 .../phoenix/end2end/BasePermissionsIT.java      |   4 +-
 .../phoenix/end2end/DropTableWithViewsIT.java   | 151 ++++++++++
 .../end2end/QueryDatabaseMetaDataIT.java        |   4 +
 .../end2end/TenantSpecificTablesDDLIT.java      |   4 +-
 .../coprocessor/MetaDataEndpointImpl.java       |  46 ++-
 .../phoenix/coprocessor/TaskRegionObserver.java | 292 +++++++++++++++++++
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |   9 +-
 .../query/ConnectionQueryServicesImpl.java      |  20 +-
 .../query/ConnectionlessQueryServicesImpl.java  |   9 +
 .../apache/phoenix/query/QueryConstants.java    |  17 +-
 .../org/apache/phoenix/query/QueryServices.java |   6 +
 .../phoenix/query/QueryServicesOptions.java     |   4 +
 .../java/org/apache/phoenix/schema/PTable.java  |  31 +-
 .../phoenix/schema/stats/StatisticsUtil.java    |   2 +
 .../org/apache/phoenix/util/SchemaUtil.java     |  10 +
 .../java/org/apache/phoenix/query/BaseTest.java |   1 +
 16 files changed, 589 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
index 88a942e..932ce9f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
@@ -428,7 +428,7 @@ public class BasePermissionsIT extends BaseTest {
             @Override
             public Object run() throws Exception {
                 try (Connection conn = getConnection(); Statement stmt = conn.createStatement();) {
-                    assertFalse(stmt.execute("DROP TABLE IF EXISTS " + tableName));
+                    assertFalse(stmt.execute(String.format("DROP TABLE IF EXISTS %s CASCADE", tableName)));
                 }
                 return null;
             }
@@ -653,7 +653,7 @@ public class BasePermissionsIT extends BaseTest {
             @Override
             public Object run() throws Exception {
                 try (Connection conn = getConnection(); Statement stmt = conn.createStatement();) {
-                    assertFalse(stmt.execute("DROP VIEW " + viewName));
+                    assertFalse(stmt.execute(String.format("DROP VIEW %s CASCADE", viewName)));
                 }
                 return null;
             }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/it/java/org/apache/phoenix/end2end/DropTableWithViewsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DropTableWithViewsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DropTableWithViewsIT.java
new file mode 100644
index 0000000..9502218
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DropTableWithViewsIT.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.phoenix.coprocessor.TableViewFinderResult;
+import org.apache.phoenix.coprocessor.ViewFinder;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.util.SchemaUtil;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class DropTableWithViewsIT extends SplitSystemCatalogIT {
+
+    private final boolean isMultiTenant;
+    private final boolean columnEncoded;
+    private final String TENANT_SPECIFIC_URL1 = getUrl() + ';' + TENANT_ID_ATTRIB + "=" + TENANT1;
+
+    public DropTableWithViewsIT(boolean isMultiTenant, boolean columnEncoded) {
+        this.isMultiTenant = isMultiTenant;
+        this.columnEncoded = columnEncoded;
+    }
+
+    @Parameters(name="DropTableWithViewsIT_multiTenant={0}, columnEncoded={1}") // name is used by failsafe as file name in reports
+    public static Collection<Boolean[]> data() {
+        return Arrays.asList(new Boolean[][] {
+                { false, false }, { false, true },
+                { true, false }, { true, true } });
+    }
+
+    private String generateDDL(String format) {
+        return generateDDL("", format);
+    }
+
+    private String generateDDL(String options, String format) {
+        StringBuilder optionsBuilder = new StringBuilder(options);
+        if (!columnEncoded) {
+            if (optionsBuilder.length() != 0)
+                optionsBuilder.append(",");
+            optionsBuilder.append("COLUMN_ENCODED_BYTES=0");
+        }
+        if (isMultiTenant) {
+            if (optionsBuilder.length() !=0 )
+                optionsBuilder.append(",");
+            optionsBuilder.append("MULTI_TENANT=true");
+        }
+        return String.format(format, isMultiTenant ? "TENANT_ID VARCHAR NOT NULL, " : "",
+            isMultiTenant ? "TENANT_ID, " : "", optionsBuilder.toString());
+    }
+
+    @Test
+    public void testDropTableWithChildViews() throws Exception {
+        String baseTable = SchemaUtil.getTableName(SCHEMA1, generateUniqueName());
+        try (Connection conn = DriverManager.getConnection(getUrl());
+                Connection viewConn =
+                        isMultiTenant ? DriverManager.getConnection(TENANT_SPECIFIC_URL1) : conn) {
+            String ddlFormat =
+                    "CREATE TABLE IF NOT EXISTS " + baseTable + "  ("
+                            + " %s PK2 VARCHAR NOT NULL, V1 VARCHAR, V2 VARCHAR "
+                            + " CONSTRAINT NAME_PK PRIMARY KEY (%s PK2)" + " ) %s";
+            conn.createStatement().execute(generateDDL(ddlFormat));
+            conn.commit();
+            // Create a view tree (i.e., tree of views) with depth of 2 and fanout factor of 4
+            for (int  i = 0; i < 4; i++) {
+                String childView = SchemaUtil.getTableName(SCHEMA2, generateUniqueName());
+                String childViewDDL = "CREATE VIEW " + childView + " AS SELECT * FROM " + baseTable;
+                viewConn.createStatement().execute(childViewDDL);
+                for (int j = 0; j < 4; j++) {
+                    String grandChildView = SchemaUtil.getTableName(SCHEMA2, generateUniqueName());
+                    String grandChildViewDDL = "CREATE VIEW " + grandChildView + " AS SELECT * FROM " + childView;
+                    viewConn.createStatement().execute(grandChildViewDDL);
+                }
+            }
+            // Drop the base table
+            String dropTable = String.format("DROP TABLE IF EXISTS %s CASCADE", baseTable);
+            conn.createStatement().execute(dropTable);
+
+            // Wait for the tasks for dropping child views to complete. The depth of the view tree is 2, so we expect that
+            // this will be done in two task handling runs, i.e., in tree task handling interval at most in general
+            // by assuming that each non-root level will be processed in one interval. To be on the safe side, we will
+            // wait at most 10 intervals.
+            long halfTimeInterval = config.getLong(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB,
+                    QueryServicesOptions.DEFAULT_TASK_HANDLING_INTERVAL_MS)/2;
+            ResultSet rs = null;
+            boolean timedOut = true;
+            Thread.sleep(3 * halfTimeInterval);
+            for (int i = 3; i < 20; i++) {
+                rs = conn.createStatement().executeQuery("SELECT * " +
+                                " FROM " + PhoenixDatabaseMetaData.SYSTEM_TASK_NAME +
+                                " WHERE " + PhoenixDatabaseMetaData.TASK_TYPE + " = " +
+                                PTable.TaskType.DROP_CHILD_VIEWS.getSerializedValue());
+                Thread.sleep(halfTimeInterval);
+                if (!rs.next()) {
+                    timedOut = false;
+                    break;
+                }
+            }
+            if (timedOut) {
+                fail("Drop child view task execution timed out!");
+            }
+            // Views should be dropped by now
+            TableName linkTable = TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES);
+            TableViewFinderResult childViewsResult = new TableViewFinderResult();
+            ViewFinder.findAllRelatives(getUtility().getConnection().getTable(linkTable),
+                    HConstants.EMPTY_BYTE_ARRAY,
+                    SchemaUtil.getSchemaNameFromFullName(baseTable).getBytes(),
+                    SchemaUtil.getTableNameFromFullName(baseTable).getBytes(),
+                    PTable.LinkType.CHILD_TABLE,
+                    childViewsResult);
+            assertTrue(childViewsResult.getLinks().size() == 0);
+            // There should not be any orphan views
+            rs = conn.createStatement().executeQuery("SELECT * FROM " + PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME +
+                    " WHERE " + PhoenixDatabaseMetaData.TABLE_SCHEM + " = '" + SCHEMA2 +"'");
+            assertFalse(rs.next());
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
index ab22b6d..226a2cc 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
@@ -186,6 +186,10 @@ public class QueryDatabaseMetaDataIT extends ParallelStatsDisabledIT {
             assertEquals(PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE, rs.getString("TABLE_NAME"));
             assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
             assertTrue(rs.next());
+            assertEquals(SYSTEM_CATALOG_SCHEMA, rs.getString("TABLE_SCHEM"));
+            assertEquals(PhoenixDatabaseMetaData.SYSTEM_TASK_TABLE, rs.getString("TABLE_NAME"));
+            assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
+            assertTrue(rs.next());
             assertEquals(null, rs.getString("TABLE_SCHEM"));
             assertEquals(tableAName, rs.getString("TABLE_NAME"));
             assertEquals(PTableType.TABLE.toString(), rs.getString("TABLE_TYPE"));

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
index 956b43c..85c9128 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
@@ -503,8 +503,10 @@ public class TenantSpecificTablesDDLIT extends BaseTenantSpecificTablesIT {
             assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.TYPE_SEQUENCE, PTableType.SYSTEM);
             assertTrue(rs.next());
             assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE, PTableType.SYSTEM);
+            assertTrue(rs.next());
+            assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_TASK_TABLE, PTableType.SYSTEM);
             assertFalse(rs.next());
-            
+
             rs = meta.getTables(null, "", StringUtil.escapeLike(tenantTable2), new String[] {TABLE.getValue().getString()});
             assertFalse(rs.next());
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index aa78b1b..424b6d6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -86,6 +86,7 @@ import java.security.PrivilegedExceptionAction;
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -1936,7 +1937,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                 table = loadTable(env, tableKey, cacheKey, clientTimeStamp, HConstants.LATEST_TIMESTAMP,
                         clientVersion);
             } catch (ParentTableNotFoundException e) {
-                dropChildMetadata(e.getParentSchemaName(), e.getParentTableName(), e.getParentTenantId());
+                dropChildViews(env, e.getParentTenantId(), e.getParentSchemaName(), e.getParentTableName());
             }
             if (table != null) {
                 if (table.getTimeStamp() < clientTimeStamp) {
@@ -1961,7 +1962,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
             // check if the table was dropped, but had child views that were have not yet
             // been cleaned up by compaction
             if (!Bytes.toString(schemaName).equals(QueryConstants.SYSTEM_SCHEMA_NAME)) {
-                dropChildMetadata(schemaName, tableName, tenantIdBytes);
+                dropChildViews(env, tenantIdBytes, schemaName, tableName);
             }
             
             byte[] parentTableKey = null;
@@ -2328,19 +2329,31 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
         }
     }
 
-    private void dropChildMetadata(byte[] schemaName, byte[] tableName, byte[] tenantIdBytes)
+    public static void dropChildViews(RegionCoprocessorEnvironment env, byte[] tenantIdBytes, byte[] schemaName, byte[] tableName)
             throws IOException, SQLException, ClassNotFoundException {
-        TableViewFinderResult childViewsResult = new TableViewFinderResult();
-        findAllChildViews(tenantIdBytes, schemaName, tableName, childViewsResult);
+        Table hTable =
+                ServerUtil.getHTableForCoprocessorScan(env,
+                        SchemaUtil.getPhysicalTableName(
+                                PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES,
+                                env.getConfiguration()).getName());
+        TableViewFinderResult childViewsResult = ViewFinder.findRelatedViews(hTable, tenantIdBytes, schemaName, tableName,
+                PTable.LinkType.CHILD_TABLE, HConstants.LATEST_TIMESTAMP);
+
         if (childViewsResult.hasLinks()) {
+
             for (TableInfo viewInfo : childViewsResult.getLinks()) {
                 byte[] viewTenantId = viewInfo.getTenantId();
                 byte[] viewSchemaName = viewInfo.getSchemaName();
                 byte[] viewName = viewInfo.getTableName();
+                if (logger.isDebugEnabled()) {
+                    logger.debug("dropChildViews :" + Bytes.toString(schemaName) + "." + Bytes.toString(tableName) +
+                            " -> " + Bytes.toString(viewSchemaName) + "." + Bytes.toString(viewName) +
+                            "with tenant id :" + Bytes.toString(viewTenantId));
+                }
                 Properties props = new Properties();
                 if (viewTenantId != null && viewTenantId.length != 0)
                     props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, Bytes.toString(viewTenantId));
-                try (PhoenixConnection connection = QueryUtil.getConnectionOnServer(env.getConfiguration())
+                try (PhoenixConnection connection = QueryUtil.getConnectionOnServer(props, env.getConfiguration())
                         .unwrap(PhoenixConnection.class)) {
                     MetaDataClient client = new MetaDataClient(connection);
                     org.apache.phoenix.parse.TableName viewTableName = org.apache.phoenix.parse.TableName
@@ -2351,7 +2364,6 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
             }
         }
     }
-
     private boolean execeededIndexQuota(PTableType tableType, PTable parentTable) {
         return PTableType.INDEX == tableType && parentTable.getIndexes().size() >= maxIndexesPerTable;
     }
@@ -2514,6 +2526,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                     }
                     throw new IllegalStateException(msg);
                 }
+
                 // drop rows from catalog on this region
                 mutateRowsWithLocks(region, localMutations, Collections.<byte[]> emptySet(), HConstants.NO_NONCE,
                         HConstants.NO_NONCE);
@@ -2631,7 +2644,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                         EnvironmentEdgeManager.currentTimeMillis(), null);
             }
 
-            if (tableType == PTableType.TABLE || tableType == PTableType.SYSTEM) {
+            if (tableType == PTableType.TABLE || tableType == PTableType.VIEW || tableType == PTableType.SYSTEM) {
                 // check to see if the table has any child views
                 try (Table hTable =
                         env.getTable(SchemaUtil.getPhysicalTableName(
@@ -2640,10 +2653,19 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                     boolean hasChildViews =
                             ViewFinder.hasChildViews(hTable, tenantId, schemaName, tableName,
                                 clientTimeStamp);
-                    if (hasChildViews && !isCascade) {
-                        // DROP without CASCADE on tables with child views is not permitted
-                        return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION,
-                                EnvironmentEdgeManager.currentTimeMillis(), null);
+                    if (hasChildViews) {
+                        if (!isCascade) {
+                            // DROP without CASCADE on tables with child views is not permitted
+                            return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION,
+                                    EnvironmentEdgeManager.currentTimeMillis(), null);
+                        }
+                        try {
+                            PhoenixConnection conn = QueryUtil.getConnectionOnServer(env.getConfiguration()).unwrap(PhoenixConnection.class);
+                            TaskRegionObserver.addTask(conn, PTable.TaskType.DROP_CHILD_VIEWS, Bytes.toString(tenantId),
+                                Bytes.toString(schemaName), Bytes.toString(tableName), this.accessCheckEnabled);
+                        } catch (Throwable t) {
+                            logger.error("Adding a task to drop child views failed!", t);
+                        }
                     }
                 }
             }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
new file mode 100644
index 0000000..ca71961
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.coprocessor;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.sql.Types;
+
+import java.util.Properties;
+import java.util.TimerTask;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import javax.annotation.concurrent.GuardedBy;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
+import org.apache.hadoop.hbase.ipc.RpcServer.Call;
+import org.apache.hadoop.hbase.ipc.RpcUtil;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.MetaDataClient;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.TaskType;
+
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.QueryUtil;
+
+
+/**
+ * Coprocessor for task related operations. This coprocessor would only be registered
+ * to SYSTEM.TASK table
+ */
+
+public class TaskRegionObserver extends BaseRegionObserver {
+    public static final Log LOG = LogFactory.getLog(TaskRegionObserver.class);
+    protected ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(TaskType.values().length);
+    private long timeInterval = QueryServicesOptions.DEFAULT_TASK_HANDLING_INTERVAL_MS;
+    private long timeMaxInterval = QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS;
+    @GuardedBy("TaskRegionObserver.class")
+    // initial delay before the first task is handled
+    private static final long  initialDelay = 10000; // 10 secs
+
+    @Override
+    public void preClose(final ObserverContext<RegionCoprocessorEnvironment> c,
+            boolean abortRequested) {
+        executor.shutdownNow();
+    }
+
+    @Override
+    public void start(CoprocessorEnvironment env) throws IOException {
+        Configuration config = env.getConfiguration();
+        timeInterval =
+                config.getLong(
+                    QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB,
+                    QueryServicesOptions.DEFAULT_TASK_HANDLING_INTERVAL_MS);
+        timeMaxInterval =
+                config.getLong(
+                        QueryServices.TASK_HANDLING_MAX_INTERVAL_MS_ATTRIB,
+                        QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS);
+    }
+
+    @Override
+    public void postOpen(ObserverContext<RegionCoprocessorEnvironment> e) {
+        final RegionCoprocessorEnvironment env = e.getEnvironment();
+
+        // turn off verbose deprecation logging
+        Logger deprecationLogger = Logger.getLogger("org.apache.hadoop.conf.Configuration.deprecation");
+        if (deprecationLogger != null) {
+            deprecationLogger.setLevel(Level.WARN);
+        }
+
+        DropChildViewsTask task = new DropChildViewsTask(e.getEnvironment(), timeMaxInterval);
+        executor.scheduleWithFixedDelay(task, initialDelay, timeInterval, TimeUnit.MILLISECONDS);
+    }
+
+    private static void mutateSystemTaskTable(final PhoenixConnection conn, final PreparedStatement stmt, boolean accessCheckEnabled)
+            throws IOException {
+        // we need to mutate SYSTEM.TASK with HBase/login user if access is enabled.
+        if (accessCheckEnabled) {
+            User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
+                @Override
+                public Void run() throws Exception {
+                    final Call rpcContext = RpcUtil.getRpcContext();
+                    // setting RPC context as null so that user can be reset
+                    try {
+                        RpcUtil.setRpcContext(null);
+                        stmt.execute();
+                        conn.commit();
+                    } catch (SQLException e) {
+                        throw new IOException(e);
+                    } finally {
+                      // setting RPC context back to original context of the RPC
+                      RpcUtil.setRpcContext(rpcContext);
+                    }
+                    return null;
+                }
+            });
+        }
+        else {
+            try {
+                stmt.execute();
+                conn.commit();
+            } catch (SQLException e) {
+                throw new IOException(e);
+            }
+        }
+    }
+
+    public static void addTask(PhoenixConnection conn, TaskType taskType, String tenantId, String schemaName,
+                               String tableName, boolean accessCheckEnabled)
+            throws IOException {
+        PreparedStatement stmt = null;
+        try {
+            stmt = conn.prepareStatement("UPSERT INTO " +
+                    PhoenixDatabaseMetaData.SYSTEM_TASK_NAME + " ( " +
+                    PhoenixDatabaseMetaData.TASK_TYPE + ", " +
+                    PhoenixDatabaseMetaData.TENANT_ID + ", " +
+                    PhoenixDatabaseMetaData.TABLE_SCHEM + ", " +
+                    PhoenixDatabaseMetaData.TABLE_NAME + " ) VALUES(?,?,?,?)");
+            stmt.setByte(1, taskType.getSerializedValue());
+            if (tenantId != null) {
+                stmt.setString(2, tenantId);
+            } else {
+                stmt.setNull(2, Types.VARCHAR);
+            }
+            if (schemaName != null) {
+                stmt.setString(3, schemaName);
+            } else {
+                stmt.setNull(3, Types.VARCHAR);
+            }
+            stmt.setString(4, tableName);
+        } catch (SQLException e) {
+            throw new IOException(e);
+        }
+        mutateSystemTaskTable(conn, stmt, accessCheckEnabled);
+    }
+
+    public static void deleteTask(PhoenixConnection conn, TaskType taskType, Timestamp ts, String tenantId,
+                                  String schemaName, String tableName, boolean accessCheckEnabled) throws IOException {
+        PreparedStatement stmt = null;
+        try {
+            stmt = conn.prepareStatement("DELETE FROM " +
+                    PhoenixDatabaseMetaData.SYSTEM_TASK_NAME +
+                    " WHERE " + PhoenixDatabaseMetaData.TASK_TYPE + " = ? AND " +
+                    PhoenixDatabaseMetaData.TASK_TS + " = ? AND " +
+                    PhoenixDatabaseMetaData.TENANT_ID + (tenantId == null ? " IS NULL " : " = '" + tenantId + "'") + " AND " +
+                    PhoenixDatabaseMetaData.TABLE_SCHEM + (schemaName == null ? " IS NULL " : " = '" + schemaName + "'") + " AND " +
+                    PhoenixDatabaseMetaData.TABLE_NAME + " = ?");
+            stmt.setByte(1, taskType.getSerializedValue());
+            stmt.setTimestamp(2, ts);
+            stmt.setString(3, tableName);
+        } catch (SQLException e) {
+            throw new IOException(e);
+        }
+        mutateSystemTaskTable(conn, stmt, accessCheckEnabled);
+    }
+
+    /**
+     * Task runs periodically to clean up task of child views whose parent is dropped
+     *
+     */
+    public static class DropChildViewsTask extends TimerTask {
+        private RegionCoprocessorEnvironment env;
+        private long timeMaxInterval;
+        private boolean accessCheckEnabled;
+
+        public DropChildViewsTask(RegionCoprocessorEnvironment env, long timeMaxInterval) {
+            this.env = env;
+            this.accessCheckEnabled = env.getConfiguration().getBoolean(QueryServices.PHOENIX_ACLS_ENABLED,
+                    QueryServicesOptions.DEFAULT_PHOENIX_ACLS_ENABLED);
+            this.timeMaxInterval = timeMaxInterval;
+        }
+
+        @Override
+        public void run() {
+            PhoenixConnection connForTask = null;
+            Timestamp timestamp = null;
+            String tenantId = null;
+            byte[] tenantIdBytes;
+            String schemaName= null;
+            byte[] schemaNameBytes;
+            String tableName = null;
+            byte[] tableNameBytes;
+            PhoenixConnection pconn;
+            try {
+                String taskQuery = "SELECT " +
+                        PhoenixDatabaseMetaData.TASK_TS + ", " +
+                        PhoenixDatabaseMetaData.TENANT_ID + ", " +
+                        PhoenixDatabaseMetaData.TABLE_SCHEM + ", " +
+                        PhoenixDatabaseMetaData.TABLE_NAME +
+                        " FROM " + PhoenixDatabaseMetaData.SYSTEM_TASK_NAME +
+                        " WHERE "+ PhoenixDatabaseMetaData.TASK_TYPE + " = " + PTable.TaskType.DROP_CHILD_VIEWS.getSerializedValue();
+
+                connForTask = QueryUtil.getConnectionOnServer(env.getConfiguration()).unwrap(PhoenixConnection.class);
+                PreparedStatement taskStatement = connForTask.prepareStatement(taskQuery);
+                ResultSet rs = taskStatement.executeQuery();
+                while (rs.next()) {
+                    try {
+                        // delete child views only if the parent table is deleted from the system catalog
+                        timestamp = rs.getTimestamp(1);
+                        tenantId = rs.getString(2);
+                        tenantIdBytes= rs.getBytes(2);
+                        schemaName= rs.getString(3);
+                        schemaNameBytes = rs.getBytes(3);
+                        tableName= rs.getString(4);
+                        tableNameBytes = rs.getBytes(4);
+
+                        if (tenantId != null) {
+                            Properties tenantProps = new Properties();
+                            tenantProps.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
+                            pconn = QueryUtil.getConnectionOnServer(tenantProps, env.getConfiguration()).unwrap(PhoenixConnection.class);
+
+                        }
+                        else {
+                            pconn = QueryUtil.getConnectionOnServer(env.getConfiguration()).unwrap(PhoenixConnection.class);
+                        }
+
+                        MetaDataProtocol.MetaDataMutationResult result = new MetaDataClient(pconn).updateCache(pconn.getTenantId(),
+                                schemaName, tableName, true);
+                        if (result.getMutationCode() != MetaDataProtocol.MutationCode.TABLE_ALREADY_EXISTS) {
+                            MetaDataEndpointImpl.dropChildViews(env, tenantIdBytes, schemaNameBytes, tableNameBytes);
+                        } else if (System.currentTimeMillis() < timeMaxInterval + timestamp.getTime()) {
+                            // skip this task as it has not been expired and its parent table has not been dropped yet
+                            LOG.info("Skipping a child view drop task. The parent table has not been dropped yet : " +
+                                    schemaName + "." + tableName +
+                                    " with tenant id " + (tenantId == null ? " IS NULL" : tenantId) +
+                                    " and timestamp " + timestamp.toString());
+                            continue;
+                        }
+                        else {
+                            LOG.warn(" A drop child view task has expired and will be removed from the system task table : " +
+                                    schemaName + "." + tableName +
+                                    " with tenant id " + (tenantId == null ? " IS NULL" : tenantId) +
+                                    " and timestamp " + timestamp.toString());
+                        }
+
+                        deleteTask(connForTask, PTable.TaskType.DROP_CHILD_VIEWS, timestamp, tenantId, schemaName,
+                                tableName, this.accessCheckEnabled);
+                    }
+                    catch (Throwable t) {
+                        LOG.warn("Exception while dropping a child view task. " +
+                                "It will be retried in the next system task table scan : " +
+                                schemaName + "." + tableName +
+                                " with tenant id " + (tenantId == null ? " IS NULL" : tenantId) +
+                                " and timestamp " + timestamp.toString(), t);
+                    }
+                }
+            } catch (Throwable t) {
+                LOG.error("DropChildViewsTask failed!", t);
+            } finally {
+                if (connForTask != null) {
+                    try {
+                        connForTask.close();
+                    } catch (SQLException ignored) {
+                        LOG.debug("DropChildViewsTask can't close connection", ignored);
+                    }
+                }
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index 52dfe99..3ff62e2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -213,6 +213,10 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
     public static final byte[] INDEX_TYPE_BYTES = Bytes.toBytes(INDEX_TYPE);
     public static final String LINK_TYPE = "LINK_TYPE";
     public static final byte[] LINK_TYPE_BYTES = Bytes.toBytes(LINK_TYPE);
+    public static final String TASK_TYPE = "TASK_TYPE";
+    public static final byte[] TASK_TYPE_BYTES = Bytes.toBytes(TASK_TYPE);
+    public static final String TASK_TS = "TASK_TS";
+    public static final byte[] TASK_TS_BYTES = Bytes.toBytes(TASK_TS);
     public static final String ARRAY_SIZE = "ARRAY_SIZE";
     public static final byte[] ARRAY_SIZE_BYTES = Bytes.toBytes(ARRAY_SIZE);
     public static final String VIEW_CONSTANT = "VIEW_CONSTANT";
@@ -361,7 +365,10 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
     public static final byte[] SYSTEM_CHILD_LINK_NAME_BYTES = Bytes.toBytes(SYSTEM_CHILD_LINK_NAME);
     public static final TableName SYSTEM_LINK_HBASE_TABLE_NAME = TableName.valueOf(SYSTEM_CHILD_LINK_NAME);
 
-    
+    public static final String SYSTEM_TASK_TABLE = "TASK";
+    public static final String SYSTEM_TASK_NAME = SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_TASK_TABLE);
+    public static final byte[] SYSTEM_TASK_NAME_BYTES = Bytes.toBytes(SYSTEM_TASK_NAME);
+    public static final TableName SYSTEM_TASK_HBASE_TABLE_NAME = TableName.valueOf(SYSTEM_TASK_NAME);
     //SYSTEM:LOG
     public static final String SYSTEM_LOG_TABLE = "LOG";
     public static final String QUERY_ID = "QUERY_ID";

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index b68637a..ebcb7b9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -152,6 +152,7 @@ import org.apache.phoenix.coprocessor.MetaDataRegionObserver;
 import org.apache.phoenix.coprocessor.ScanRegionObserver;
 import org.apache.phoenix.coprocessor.SequenceRegionObserver;
 import org.apache.phoenix.coprocessor.ServerCachingEndpointImpl;
+import org.apache.phoenix.coprocessor.TaskRegionObserver;
 import org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver;
 import org.apache.phoenix.coprocessor.generated.MetaDataProtos;
 import org.apache.phoenix.coprocessor.generated.MetaDataProtos.AddColumnRequest;
@@ -958,7 +959,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                 if (!descriptor.hasCoprocessor(SequenceRegionObserver.class.getName())) {
                     descriptor.addCoprocessor(SequenceRegionObserver.class.getName(), null, priority, null);
                 }
+            } else if (SchemaUtil.isTaskTable(tableName)) {
+                if(!descriptor.hasCoprocessor(TaskRegionObserver.class.getName())) {
+                    descriptor.addCoprocessor(TaskRegionObserver.class.getName(), null, priority, null);
             }
+        }
 
             if (isTransactional) {
                 Class<? extends RegionObserver> coprocessorClass = provider.getTransactionProvider().getCoprocessor();
@@ -2812,6 +2817,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         return setSystemDDLProperties(QueryConstants.CREATE_MUTEX_METADTA);
     }
 
+    protected String getTaskDDL() {
+        return setSystemDDLProperties(QueryConstants.CREATE_TASK_METADATA);
+    }
+
     private String setSystemDDLProperties(String ddl) {
         return String.format(ddl,
           props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS),
@@ -3038,6 +3047,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         try {
             metaConnection.createStatement().executeUpdate(getMutexDDL());
         } catch (TableAlreadyExistsException e) {}
+        try {
+            metaConnection.createStatement().executeUpdate(getTaskDDL());
+        } catch (TableAlreadyExistsException e) {}
+
         // Catch the IOException to log the error message and then bubble it up for the client to retry.
     }
 
@@ -3496,6 +3509,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
             try {
                 metaConnection.createStatement().executeUpdate(getMutexDDL());
             } catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {}
+            try {
+                metaConnection.createStatement().executeUpdate(getTaskDDL());
+            } catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {}
 
             // In case namespace mapping is enabled and system table to system namespace mapping is also enabled,
             // create an entry for the SYSTEM namespace in the SYSCAT table, so that GRANT/REVOKE commands can work
@@ -3733,8 +3749,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
             // No tables exist matching "SYSTEM\..*", they are all already in "SYSTEM:.*"
             if (tableNames.size() == 0) { return; }
             // Try to move any remaining tables matching "SYSTEM\..*" into "SYSTEM:"
-            if (tableNames.size() > 7) {
-                logger.warn("Expected 7 system tables but found " + tableNames.size() + ":" + tableNames);
+            if (tableNames.size() > 8) {
+                logger.warn("Expected 8 system tables but found " + tableNames.size() + ":" + tableNames);
             }
 
             byte[] mappedSystemTable = SchemaUtil

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index bbfacc0..655de0d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -185,6 +185,10 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
         return setSystemDDLProperties(QueryConstants.CREATE_MUTEX_METADTA);
     }
 
+    protected String getTaskDDL() {
+        return setSystemDDLProperties(QueryConstants.CREATE_TASK_METADATA);
+    }
+
     private String setSystemDDLProperties(String ddl) {
         return String.format(ddl,
           props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS),
@@ -388,6 +392,11 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
                             .executeUpdate(getMutexDDL());
                 } catch (NewerTableAlreadyExistsException ignore) {
                 }
+                try {
+                    metaConnection.createStatement()
+                            .executeUpdate(getTaskDDL());
+                } catch (NewerTableAlreadyExistsException ignore) {
+                }
             } catch (SQLException e) {
                 sqlE = e;
             } finally {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index bbff343..c45fec9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -347,8 +347,8 @@ public interface QueryConstants {
     
 	// Links from parent to child views are stored in a separate table for
 	// scalability
-	public static final String CREATE_CHILD_LINK_METADATA = "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\""
-			+ SYSTEM_CHILD_LINK_TABLE + "\"(\n" +
+	public static final String CREATE_CHILD_LINK_METADATA = "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" +
+            SYSTEM_CHILD_LINK_TABLE + "\"(\n" +
 			// PK columns
 			TENANT_ID + " VARCHAR NULL," + TABLE_SCHEM + " VARCHAR NULL," + TABLE_NAME + " VARCHAR NOT NULL,"
 			+ COLUMN_NAME + " VARCHAR NULL," + COLUMN_FAMILY + " VARCHAR NULL," + LINK_TYPE + " UNSIGNED_TINYINT,\n"
@@ -371,4 +371,17 @@ public interface QueryConstants {
 	            HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n" +
 	            PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
     
+	public static final String CREATE_TASK_METADATA =
+            "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_TASK_TABLE + "\"(\n" +
+            // PK columns
+            TASK_TYPE + " UNSIGNED_TINYINT NOT NULL," +
+            TASK_TS + " TIMESTAMP NOT NULL," +
+            TENANT_ID + " VARCHAR NULL," +
+            TABLE_SCHEM + " VARCHAR NULL," +
+            TABLE_NAME + " VARCHAR NOT NULL,\n" +
+            "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TASK_TYPE + "," + TASK_TS + " ROW_TIMESTAMP," + TENANT_ID + "," + TABLE_SCHEM + "," +
+            TABLE_NAME + "))\n" +
+            HConstants.VERSIONS + "=%s,\n" +
+            HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n" +
+            PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 337bb05..d06f07c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -320,6 +320,12 @@ public interface QueryServices extends SQLCloseable {
 
 	public static final String SYSTEM_CATALOG_SPLITTABLE = "phoenix.system.catalog.splittable";
 
+    // The parameters defined for handling task stored in table SYSTEM.TASK
+	// The time interval between periodic scans of table SYSTEM.TASK
+    public static final String TASK_HANDLING_INTERVAL_MS_ATTRIB = "phoenix.task.handling.interval.ms";
+    // The maximum time for a task to stay in table SYSTEM.TASK
+    public static final String TASK_HANDLING_MAX_INTERVAL_MS_ATTRIB = "phoenix.task.handling.maxInterval.ms";
+
     /**
      * Get executor service used for parallel scans
      */

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 35dbe3a..a6a73f7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -340,6 +340,10 @@ public class QueryServicesOptions {
     public static final int DEFAULT_UPDATE_CACHE_FREQUENCY = 0;
     public static final int DEFAULT_SMALL_SCAN_THRESHOLD = 100;
 
+    // default system task handling interval in milliseconds
+    public static final long DEFAULT_TASK_HANDLING_INTERVAL_MS = 60*1000; // 1 min
+    public static final long DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS = 30*60*1000; // 30 minutes
+
     @SuppressWarnings("serial")
     public static final Set<String> DEFAULT_QUERY_SERVER_SKIP_WORDS = new HashSet<String>() {
       {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
index 8cbf757..6dfe411 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
@@ -192,7 +192,36 @@ public interface PTable extends PMetaDataEntity {
             return LinkType.values()[serializedValue-1];
         }
     }
-    
+
+    public enum TaskType {
+        DROP_CHILD_VIEWS((byte)1);
+
+        private final byte[] byteValue;
+        private final byte serializedValue;
+
+        TaskType(byte serializedValue) {
+            this.serializedValue = serializedValue;
+            this.byteValue = Bytes.toBytes(this.name());
+        }
+
+        public byte[] getBytes() {
+                return byteValue;
+        }
+
+        public byte getSerializedValue() {
+            return this.serializedValue;
+        }
+        public static TaskType getDefault() {
+            return DROP_CHILD_VIEWS;
+        }
+        public static TaskType fromSerializedValue(byte serializedValue) {
+            if (serializedValue < 1 || serializedValue > TaskType.values().length) {
+                    throw new IllegalArgumentException("Invalid TaskType " + serializedValue);
+            }
+            return TaskType.values()[serializedValue-1];
+        }
+    }
+
     public enum ImmutableStorageScheme implements ColumnValueEncoderDecoderSupplier {
         ONE_CELL_PER_COLUMN((byte)1) {
             @Override

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
index 4a758b7..23b1fcc 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
@@ -62,10 +62,12 @@ public class StatisticsUtil {
         DISABLE_STATS.add(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_FUNCTION_NAME));
         DISABLE_STATS.add(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME));
         DISABLE_STATS.add(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME));
+        DISABLE_STATS.add(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_TASK_NAME));
         DISABLE_STATS.add(SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES,true));
         DISABLE_STATS.add(SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_FUNCTION_NAME_BYTES,true));
         DISABLE_STATS.add(SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME_BYTES,true));
         DISABLE_STATS.add(SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES,true));
+        DISABLE_STATS.add(SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_TASK_NAME_BYTES,true));
     }
     
     private StatisticsUtil() {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
index ec2eb14..24a0e12 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
@@ -540,6 +540,12 @@ public class SchemaUtil {
                 || Bytes.compareTo(tableName, SchemaUtil
                         .getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME_BYTES, true).getName()) == 0;
     }
+
+    public static boolean isTaskTable(byte[] tableName) {
+        return Bytes.compareTo(tableName, PhoenixDatabaseMetaData.SYSTEM_TASK_NAME_BYTES) == 0
+                || Bytes.compareTo(tableName, SchemaUtil
+                .getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_TASK_NAME_BYTES, true).getName()) == 0;
+    }
     
     public static boolean isChildLinkTable(byte[] tableName) {
         return Bytes.compareTo(tableName, SYSTEM_CHILD_LINK_NAME_BYTES) == 0 || Bytes.compareTo(tableName,
@@ -550,6 +556,10 @@ public class SchemaUtil {
         return PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME.equals(table.getName().getString());
     }
 
+    public static boolean isTaskTable(PTable table) {
+        return PhoenixDatabaseMetaData.SYSTEM_TASK_NAME.equals(table.getName().getString());
+    }
+
     public static boolean isMetaTable(PTable table) {
         return PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA.equals(table.getSchemaName().getString()) && PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE.equals(table.getTableName().getString());
     }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index 5ca247b..e6e3936 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -620,6 +620,7 @@ public abstract class BaseTest {
         conf.setInt("hbase.assignment.zkevent.workers", 5);
         conf.setInt("hbase.assignment.threads.max", 5);
         conf.setInt("hbase.catalogjanitor.interval", 5000);
+        conf.setInt(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, 1000);
         conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
         conf.setInt(NUM_CONCURRENT_INDEX_WRITER_THREADS_CONF_KEY, 1);
         return conf;


[04/22] phoenix git commit: PHOENIX-4872: BulkLoad has bug when loading on single-cell-array-with-offsets table.

Posted by ja...@apache.org.
PHOENIX-4872: BulkLoad has bug when loading on single-cell-array-with-offsets table.


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/fe466ebe
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/fe466ebe
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/fe466ebe

Branch: refs/heads/omid2
Commit: fe466ebe8e546c24f686232e7f3be03f96f1448a
Parents: 527098e
Author: s.kadam <s....@gus.com>
Authored: Mon Oct 29 14:47:21 2018 -0700
Committer: Thomas D'Silva <td...@apache.org>
Committed: Tue Oct 30 14:56:22 2018 -0700

----------------------------------------------------------------------
 .../phoenix/end2end/CsvBulkLoadToolIT.java      | 51 ++++++++++++++++++++
 .../mapreduce/FormatToBytesWritableMapper.java  | 10 ++--
 2 files changed, 56 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/fe466ebe/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
index 40fe900..7e4226d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
@@ -35,7 +35,10 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.FileAlreadyExistsException;
+import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.mapreduce.CsvBulkLoadTool;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableKey;
 import org.apache.phoenix.util.DateUtil;
 import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.ReadOnlyProps;
@@ -446,4 +449,52 @@ public class CsvBulkLoadToolIT extends BaseOwnClusterIT {
         rs.close();
         stmt.close();
     }
+
+    /**
+     * This test case validates the import using CsvBulkLoadTool in
+     * SingleCellArrayWithOffsets table.
+     * PHOENIX-4872
+     */
+
+    @Test
+    public void testImportInSingleCellArrayWithOffsetsTable() throws Exception {
+        Statement stmt = conn.createStatement();
+        stmt.execute("CREATE IMMUTABLE TABLE S.TABLE12 (ID INTEGER NOT NULL PRIMARY KEY," +
+                " CF0.NAME VARCHAR, CF0.T DATE, CF1.T2 DATE, CF2.T3 DATE) " +
+                "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS");
+        PhoenixConnection phxConn = conn.unwrap(PhoenixConnection.class);
+        PTable table = phxConn.getTable(new PTableKey(null, "S.TABLE12"));
+
+        assertEquals(PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS,
+                table.getImmutableStorageScheme());
+
+        FileSystem fs = FileSystem.get(getUtility().getConfiguration());
+        FSDataOutputStream outputStream = fs.create(new Path("/tmp/inputSCAWO.csv"));
+        PrintWriter printWriter = new PrintWriter(outputStream);
+        printWriter.println("1,Name 1,1970/01/01,1970/02/01,1970/03/01");
+        printWriter.println("2,Name 2,1970/01/02,1970/02/02,1970/03/02");
+        printWriter.println("3,Name 1,1970/01/01,1970/02/03,1970/03/01");
+        printWriter.println("4,Name 2,1970/01/02,1970/02/04,1970/03/02");
+        printWriter.println("5,Name 1,1970/01/01,1970/02/05,1970/03/01");
+        printWriter.println("6,Name 2,1970/01/02,1970/02/06,1970/03/02");
+        printWriter.close();
+
+        CsvBulkLoadTool csvBulkLoadTool = new CsvBulkLoadTool();
+        csvBulkLoadTool.setConf(new Configuration(getUtility().getConfiguration()));
+        csvBulkLoadTool.getConf().set(DATE_FORMAT_ATTRIB,"yyyy/MM/dd");
+        int exitCode = csvBulkLoadTool.run(new String[] {
+                "--input", "/tmp/inputSCAWO.csv",
+                "--table", "table12",
+                "--schema", "s",
+                "--zookeeper", zkQuorum});
+        assertEquals(0, exitCode);
+
+        ResultSet rs = stmt.executeQuery("SELECT COUNT(1) FROM S.TABLE12");
+        assertTrue(rs.next());
+        assertEquals(6, rs.getInt(1));
+
+        rs.close();
+        stmt.close();
+
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fe466ebe/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
index 360859e..93ab188 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
@@ -242,12 +242,12 @@ public abstract class FormatToBytesWritableMapper<RECORD> extends Mapper<LongWri
                         columnIndex++;
                     }
                 }
-                byte[] emptyColumnFamily = SchemaUtil.getEmptyColumnFamily(table);
-                byte[] emptyKeyValue = EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst();
-                byte[] cfn = Bytes.add(emptyColumnFamily, QueryConstants.NAMESPACE_SEPARATOR_BYTES, emptyKeyValue);
-                columnIndexes.put(cfn, new Integer(columnIndex));
-                columnIndex++;
             }
+            byte[] emptyColumnFamily = SchemaUtil.getEmptyColumnFamily(table);
+            byte[] emptyKeyValue = EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst();
+            byte[] cfn = Bytes.add(emptyColumnFamily, QueryConstants.NAMESPACE_SEPARATOR_BYTES, emptyKeyValue);
+            columnIndexes.put(cfn, new Integer(columnIndex));
+            columnIndex++;
         }
     }
 


[21/22] phoenix git commit: Merge branch '4.x-HBase-1.3' into omid2

Posted by ja...@apache.org.
Merge branch '4.x-HBase-1.3' into omid2


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/cf087158
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/cf087158
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/cf087158

Branch: refs/heads/omid2
Commit: cf087158170dbf7c9d18c02bedb76f5ec91d7b23
Parents: 0f2b1b8 169270d
Author: James Taylor <ja...@apache.org>
Authored: Sat Nov 10 12:14:49 2018 -0800
Committer: James Taylor <ja...@apache.org>
Committed: Sat Nov 10 12:14:49 2018 -0800

----------------------------------------------------------------------
 bin/phoenix_utils.py                            |    2 +
 phoenix-assembly/pom.xml                        |    4 +
 .../src/build/components/all-common-jars.xml    |    8 +
 .../org/apache/phoenix/end2end/AggregateIT.java |  987 +-------------
 .../apache/phoenix/end2end/BaseAggregateIT.java | 1022 ++++++++++++++
 .../apache/phoenix/end2end/BaseOrderByIT.java   |  940 +++++++++++++
 .../phoenix/end2end/BasePermissionsIT.java      |    4 +-
 .../phoenix/end2end/CsvBulkLoadToolIT.java      |   51 +
 .../phoenix/end2end/DropTableWithViewsIT.java   |  151 +++
 .../MigrateSystemTablesToSystemNamespaceIT.java |    4 +-
 .../org/apache/phoenix/end2end/OrderByIT.java   |  943 ++-----------
 .../end2end/ParallelStatsDisabledIT.java        |   40 +
 .../phoenix/end2end/PropertiesInSyncIT.java     |  172 ++-
 .../end2end/QueryDatabaseMetaDataIT.java        |    4 +
 .../SystemCatalogCreationOnConnectionIT.java    |    4 +-
 .../end2end/TableSnapshotReadsMapReduceIT.java  |  122 +-
 .../end2end/TenantSpecificTablesDDLIT.java      |    4 +-
 .../phoenix/end2end/index/IndexMetadataIT.java  |  145 +-
 .../end2end/salted/BaseSaltedTableIT.java       |  474 +++++++
 .../phoenix/end2end/salted/SaltedTableIT.java   |  450 +------
 .../monitoring/BasePhoenixMetricsIT.java        |    4 +-
 .../org/apache/phoenix/rpc/UpdateCacheIT.java   |  134 +-
 .../apache/phoenix/compile/DeleteCompiler.java  |    5 +-
 .../apache/phoenix/compile/FromCompiler.java    |   66 +-
 .../apache/phoenix/compile/JoinCompiler.java    |   53 +-
 .../compile/TupleProjectionCompiler.java        |   60 +-
 .../apache/phoenix/compile/UnionCompiler.java   |   41 +-
 .../apache/phoenix/compile/UpsertCompiler.java  |   12 +-
 .../coprocessor/MetaDataEndpointImpl.java       |  142 +-
 .../phoenix/coprocessor/TaskRegionObserver.java |  295 ++++
 .../UngroupedAggregateRegionObserver.java       |    6 +-
 .../coprocessor/WhereConstantParser.java        |    3 +-
 .../phoenix/exception/SQLExceptionCode.java     |    7 +-
 .../index/write/DelegateIndexFailurePolicy.java |    5 +-
 .../index/PhoenixIndexFailurePolicy.java        |    6 +
 .../iterate/MapReduceParallelScanGrouper.java   |   32 +-
 .../iterate/TableSnapshotResultIterator.java    |   28 +-
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |    9 +-
 .../mapreduce/FormatToBytesWritableMapper.java  |   10 +-
 .../query/ConnectionQueryServicesImpl.java      |   28 +-
 .../query/ConnectionlessQueryServicesImpl.java  |   18 +-
 .../apache/phoenix/query/QueryConstants.java    |   17 +-
 .../org/apache/phoenix/query/QueryServices.java |    9 +
 .../phoenix/query/QueryServicesOptions.java     |    5 +
 .../apache/phoenix/schema/MetaDataClient.java   |  248 +++-
 .../apache/phoenix/schema/PMetaDataImpl.java    |   28 +-
 .../java/org/apache/phoenix/schema/PTable.java  |   31 +-
 .../org/apache/phoenix/schema/PTableImpl.java   | 1259 +++++++++++-------
 .../org/apache/phoenix/schema/TableRef.java     |   17 +-
 .../phoenix/schema/stats/StatisticsUtil.java    |    2 +
 .../org/apache/phoenix/util/MetaDataUtil.java   |    6 +-
 .../org/apache/phoenix/util/QueryBuilder.java   |  211 +++
 .../java/org/apache/phoenix/util/QueryUtil.java |   38 +-
 .../org/apache/phoenix/util/SchemaUtil.java     |   10 +
 .../org/apache/phoenix/util/UpgradeUtil.java    |   79 +-
 .../phoenix/execute/CorrelatePlanTest.java      |   32 +-
 .../execute/LiteralResultIteratorPlanTest.java  |   33 +-
 .../index/IndexScrutinyTableOutputTest.java     |    6 +-
 .../util/PhoenixConfigurationUtilTest.java      |    6 +-
 .../java/org/apache/phoenix/query/BaseTest.java |   15 +-
 .../org/apache/phoenix/util/QueryUtilTest.java  |   10 +-
 phoenix-server/pom.xml                          |    1 +
 phoenix-spark/pom.xml                           |    8 +
 .../org/apache/phoenix/spark/AggregateIT.java   |   91 ++
 .../org/apache/phoenix/spark/OrderByIT.java     |  460 +++++++
 .../org/apache/phoenix/spark/SaltedTableIT.java |   53 +
 .../org/apache/phoenix/spark/SparkUtil.java     |   87 ++
 .../apache/phoenix/spark/PhoenixSparkIT.scala   |    9 +-
 .../apache/phoenix/spark/SparkResultSet.java    | 1056 +++++++++++++++
 .../org/apache/phoenix/spark/PhoenixRDD.scala   |   27 +-
 70 files changed, 7235 insertions(+), 3114 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/cf087158/phoenix-assembly/pom.xml
----------------------------------------------------------------------
diff --cc phoenix-assembly/pom.xml
index f9d6d92,e41d7e6..9d3c891
--- a/phoenix-assembly/pom.xml
+++ b/phoenix-assembly/pom.xml
@@@ -127,19 -127,8 +127,23 @@@
        <artifactId>phoenix-queryserver-client</artifactId>
      </dependency>
      <dependency>
 +      <groupId>com.google.inject</groupId>
 +      <artifactId>guice</artifactId>
 +      <version>3.0</version>
 +    </dependency>
 +    <dependency>
 +      <groupId>org.apache.omid</groupId>
 +      <artifactId>omid-hbase-tools-hbase1.x</artifactId>
 +      <version>${omid.version}</version>
 +    </dependency>
 +    <dependency>
 +      <groupId>org.apache.htrace</groupId>
 +      <artifactId>htrace-core</artifactId>
 +      <version>3.2.0-incubating</version>
 +    </dependency>
++    <dependency>
+       <groupId>org.apache.phoenix</groupId>
+       <artifactId>phoenix-load-balancer</artifactId>
+     </dependency>
    </dependencies>
  </project>

http://git-wip-us.apache.org/repos/asf/phoenix/blob/cf087158/phoenix-server/pom.xml
----------------------------------------------------------------------


[14/22] phoenix git commit: PHOENIX-4981 Add tests for ORDER BY, GROUP BY and salted tables using phoenix-spark

Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateIT.java
new file mode 100644
index 0000000..5b466df
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateIT.java
@@ -0,0 +1,1022 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.apache.phoenix.util.TestUtil.assertResultSet;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.List;
+import java.util.Properties;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.compile.QueryPlan;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.query.KeyRange;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.AmbiguousColumnException;
+import org.apache.phoenix.schema.types.PChar;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryBuilder;
+import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.Test;
+
+
+public abstract class BaseAggregateIT extends ParallelStatsDisabledIT {
+
+    private static void initData(Connection conn, String tableName) throws SQLException {
+        conn.createStatement().execute("create table " + tableName +
+                "   (id varchar not null primary key,\n" +
+                "    uri varchar, appcpu integer)");
+        insertRow(conn, tableName, "Report1", 10, 1);
+        insertRow(conn, tableName, "Report2", 10, 2);
+        insertRow(conn, tableName, "Report3", 30, 3);
+        insertRow(conn, tableName, "Report4", 30, 4);
+        insertRow(conn, tableName, "SOQL1", 10, 5);
+        insertRow(conn, tableName, "SOQL2", 10, 6);
+        insertRow(conn, tableName, "SOQL3", 30, 7);
+        insertRow(conn, tableName, "SOQL4", 30, 8);
+        conn.commit();
+    }
+
+    private static void insertRow(Connection conn, String tableName, String uri, int appcpu, int id) throws SQLException {
+        PreparedStatement statement = conn.prepareStatement("UPSERT INTO " + tableName + "(id, uri, appcpu) values (?,?,?)");
+        statement.setString(1, "id" + id);
+        statement.setString(2, uri);
+        statement.setInt(3, appcpu);
+        statement.executeUpdate();
+    }
+
+    @Test
+    public void testDuplicateTrailingAggExpr() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        props.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.FALSE.toString());
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String tableName = generateUniqueName();
+
+        conn.createStatement().execute("create table " + tableName +
+                "   (nam VARCHAR(20), address VARCHAR(20), id BIGINT "
+                + "constraint my_pk primary key (id))");
+        PreparedStatement statement = conn.prepareStatement("UPSERT INTO " + tableName + "(nam, address, id) values (?,?,?)");
+        statement.setString(1, "pulkit");
+        statement.setString(2, "badaun");
+        statement.setInt(3, 1);
+        statement.executeUpdate();
+        conn.commit();
+
+        QueryBuilder queryBuilder = new QueryBuilder()
+                .setDistinct(true)
+                .setSelectExpression("'harshit' as TEST_COLUMN, trim(NAM), trim(NAM)")
+                .setSelectExpressionColumns(Lists.newArrayList("NAM"))
+                .setFullTableName(tableName);
+
+        ResultSet rs = executeQuery(conn, queryBuilder);
+        assertTrue(rs.next());
+        assertEquals("harshit", rs.getString(1));
+        assertEquals("pulkit", rs.getString(2));
+        assertEquals("pulkit", rs.getString(3));
+        conn.close();
+    }
+
+    @Test
+    public void testExpressionInGroupBy() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String tableName = generateUniqueName();
+        String ddl = "create table " + tableName + "(tgb_id integer NOT NULL,utc_date_epoch integer NOT NULL,tgb_name varchar(40),ack_success_count integer" +
+                ",ack_success_one_ack_count integer, CONSTRAINT pk_tgb_counter PRIMARY KEY(tgb_id, utc_date_epoch))";
+
+        createTestTable(getUrl(), ddl);
+        String dml = "UPSERT INTO " + tableName + " VALUES(?,?,?,?,?)";
+        PreparedStatement stmt = conn.prepareStatement(dml);
+        stmt.setInt(1, 1);
+        stmt.setInt(2, 1000);
+        stmt.setString(3, "aaa");
+        stmt.setInt(4, 1);
+        stmt.setInt(5, 1);
+        stmt.execute();
+        stmt.setInt(1, 2);
+        stmt.setInt(2, 2000);
+        stmt.setString(3, "bbb");
+        stmt.setInt(4, 2);
+        stmt.setInt(5, 2);
+        stmt.execute();
+        conn.commit();
+
+        QueryBuilder queryBuilder = new QueryBuilder()
+            .setSelectExpression("TGB_ID, TGB_NAME, (UTC_DATE_EPOCH/10)*10 AS UTC_EPOCH_HOUR,SUM(ACK_SUCCESS_COUNT + " +
+                    "ACK_SUCCESS_ONE_ACK_COUNT) AS ACK_TX_SUM")
+            .setSelectExpressionColumns(Lists.newArrayList("TGB_ID", "TGB_NAME",
+                "UTC_DATE_EPOCH", "ACK_SUCCESS_COUNT", "ACK_SUCCESS_ONE_ACK_COUNT"))
+            .setGroupByClause("TGB_ID, TGB_NAME, UTC_EPOCH_HOUR")
+            .setFullTableName(tableName)
+            .setOrderByClause("TGB_ID, UTC_EPOCH_HOUR");
+
+        ResultSet rs = executeQuery(conn, queryBuilder);
+        assertTrue(rs.next());
+        assertEquals(1,rs.getInt(1));
+        assertEquals("aaa",rs.getString(2));
+        assertEquals(1000,rs.getDouble(3), 1e-6);
+        assertEquals(2,rs.getLong(4));
+        assertTrue(rs.next());
+        assertEquals(2,rs.getInt(1));
+        assertEquals("bbb",rs.getString(2));
+        assertEquals(2000,rs.getDouble(3), 1e-6);
+        assertEquals(4,rs.getLong(4));
+        assertFalse(rs.next());
+        rs.close();
+        conn.close();
+    }
+    
+    @Test
+    public void testBooleanInGroupBy() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String tableName = generateUniqueName();
+        String ddl = " create table " + tableName + "(id varchar primary key,v1 boolean, v2 integer, v3 integer)";
+
+        createTestTable(getUrl(), ddl);
+        PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName + "(id,v2,v3) VALUES(?,?,?)");
+        stmt.setString(1, "a");
+        stmt.setInt(2, 1);
+        stmt.setInt(3, 1);
+        stmt.execute();
+        stmt.close();
+        stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES(?,?,?,?)");
+        stmt.setString(1, "b");
+        stmt.setBoolean(2, false);
+        stmt.setInt(3, 2);
+        stmt.setInt(4, 2);
+        stmt.execute();
+        stmt.setString(1, "c");
+        stmt.setBoolean(2, true);
+        stmt.setInt(3, 3);
+        stmt.setInt(4, 3);
+        stmt.execute();
+        conn.commit();
+
+        String[] gbs = {"V1,V2,V3","V1,V3,V2","V2,V1,V3"};
+        for (String gb : gbs) {
+            QueryBuilder queryBuilder = new QueryBuilder()
+                .setSelectColumns(Lists.newArrayList("V1", "V2", "V3"))
+                .setFullTableName(tableName)
+                .setGroupByClause(gb);
+
+            ResultSet rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertEquals(false,rs.getBoolean("v1"));
+            assertTrue(rs.wasNull());
+            assertEquals(1,rs.getInt("v2"));
+            assertEquals(1,rs.getInt("v3"));
+            assertTrue(rs.next());
+            assertEquals(false,rs.getBoolean("v1"));
+            assertFalse(rs.wasNull());
+            assertEquals(2,rs.getInt("v2"));
+            assertEquals(2,rs.getInt("v3"));
+            assertTrue(rs.next());
+            assertEquals(true,rs.getBoolean("v1"));
+            assertEquals(3,rs.getInt("v2"));
+            assertEquals(3,rs.getInt("v3"));
+            assertFalse(rs.next());
+            rs.close();
+        }
+        conn.close();
+    }
+    
+    @Test
+    public void testScanUri() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String tableName = generateUniqueName();
+        initData(conn, tableName);
+        Statement stmt = conn.createStatement();
+        QueryBuilder queryBuilder = new QueryBuilder()
+            .setSelectColumns(Lists.newArrayList("URI"))
+            .setFullTableName(tableName);
+
+        ResultSet rs = executeQuery(conn, queryBuilder);
+        assertTrue(rs.next());
+        assertEquals("Report1", rs.getString(1));
+        assertTrue(rs.next());
+        assertEquals("Report2", rs.getString(1));
+        assertTrue(rs.next());
+        assertEquals("Report3", rs.getString(1));
+        assertTrue(rs.next());
+        assertEquals("Report4", rs.getString(1));
+        assertTrue(rs.next());
+        assertEquals("SOQL1", rs.getString(1));
+        assertTrue(rs.next());
+        assertEquals("SOQL2", rs.getString(1));
+        assertTrue(rs.next());
+        assertEquals("SOQL3", rs.getString(1));
+        assertTrue(rs.next());
+        assertEquals("SOQL4", rs.getString(1));
+        assertFalse(rs.next());
+        conn.close();
+    }
+
+    @Test
+    public void testCount() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String tableName = generateUniqueName();
+        initData(conn, tableName);
+        Statement stmt = conn.createStatement();
+        QueryBuilder queryBuilder = new QueryBuilder()
+            .setSelectExpression("count(1)")
+            .setFullTableName(tableName);
+        ResultSet rs = executeQuery(conn, queryBuilder);
+        assertTrue(rs.next());
+        assertEquals(8, rs.getLong(1));
+        assertFalse(rs.next());
+        conn.close();
+    }
+
+    @Test
+    public void testGroupByCase() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        props.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.FALSE.toString());
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String tableName = generateUniqueName();
+        initData(conn, tableName);
+        QueryBuilder queryBuilder = new QueryBuilder()
+            .setSelectExpression("CASE WHEN URI LIKE 'REPORT%' THEN 'REPORTS' ELSE 'OTHER' END CATEGORY, AVG(APPCPU)")
+            .setSelectExpressionColumns(Lists.newArrayList("URI", "APPCPU"))
+            .setFullTableName(tableName)
+            .setGroupByClause("CATEGORY");
+        executeQuery(conn, queryBuilder);
+
+        queryBuilder.setSelectExpression(
+                "CASE URI WHEN 'REPORT%' THEN 'REPORTS' ELSE 'OTHER' END CATEGORY, AVG(APPCPU)")
+            .setSelectExpressionColumns(Lists.newArrayList("URI", "APPCPU"))
+            .setFullTableName(tableName)
+            .setGroupByClause("APPCPU, CATEGORY");
+        executeQuery(conn, queryBuilder);
+
+        queryBuilder.setSelectExpression(
+                "CASE URI WHEN 'Report%' THEN 'Reports' ELSE 'Other' END CATEGORY, AVG(APPCPU)")
+            .setSelectColumns(Lists.newArrayList("URI", "APPCPU"))
+            .setFullTableName(tableName)
+            .setGroupByClause("AVG(APPCPU), CATEGORY");
+        executeQueryThrowsException(conn, queryBuilder, "Aggregate expressions may not be used in GROUP BY", "");
+        conn.close();
+    }
+
+
+    @Test
+    public void testGroupByArray() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+
+        String tableName = generateUniqueName();
+        conn.createStatement().execute("CREATE TABLE " + tableName + "(\n" + 
+                "  a VARCHAR NOT NULL,\n" + 
+                "  b VARCHAR,\n" + 
+                "  c INTEGER,\n" + 
+                "  d VARCHAR,\n" + 
+                "  e VARCHAR ARRAY,\n" + 
+                "  f BIGINT,\n" + 
+                "  g BIGINT,\n" + 
+                "  CONSTRAINT pk PRIMARY KEY(a)\n" + 
+                ")");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('1', 'val', 100, 'a', ARRAY ['b'], 1, 2)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('2', 'val', 100, 'a', ARRAY ['b'], 3, 4)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('3', 'val', 100, 'a', ARRAY ['b','c'], 5, 6)");
+        conn.commit();
+        
+        QueryBuilder queryBuilder = new QueryBuilder()
+            .setSelectExpression("C, SUM(F + G) AS SUMONE, D, E")
+            .setSelectExpressionColumns(Lists.newArrayList("A", "B", "C", "F", "G", "D", "E"))
+            .setWhereClause("B = 'val' AND A IN ('1','2','3')")
+            .setFullTableName(tableName)
+            .setGroupByClause("C, D, E")
+            .setOrderByClause("SUMONE desc");
+        ResultSet rs = executeQuery(conn, queryBuilder);
+        assertTrue(rs.next());
+        assertEquals(100, rs.getInt(1));
+        assertEquals(11, rs.getLong(2));
+        assertTrue(rs.next());
+        assertEquals(100, rs.getInt(1));
+        assertEquals(10, rs.getLong(2));
+        assertFalse(rs.next());
+        conn.close();
+    }
+    
+    @Test
+    public void testGroupByOrderPreserving() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        props.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.FALSE.toString());
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String tableName = generateUniqueName();
+
+        conn.createStatement().execute("CREATE TABLE " + tableName + "(ORGANIZATION_ID char(15) not null, \n" + 
+                "JOURNEY_ID char(15) not null, \n" + 
+                "DATASOURCE SMALLINT not null, \n" + 
+                "MATCH_STATUS TINYINT not null, \n" + 
+                "EXTERNAL_DATASOURCE_KEY varchar(30), \n" + 
+                "ENTITY_ID char(15) not null, \n" + 
+                "CONSTRAINT PK PRIMARY KEY (\n" + 
+                "    ORGANIZATION_ID, \n" + 
+                "    JOURNEY_ID, \n" + 
+                "    DATASOURCE, \n" + 
+                "    MATCH_STATUS,\n" + 
+                "    EXTERNAL_DATASOURCE_KEY,\n" + 
+                "    ENTITY_ID))");
+        conn.createStatement().execute("UPSERT INTO " + tableName
+                + " VALUES('000001111122222', '333334444455555', 0, 0, 'abc', '666667777788888')");
+        conn.createStatement().execute("UPSERT INTO " + tableName
+                + " VALUES('000001111122222', '333334444455555', 0, 0, 'abcd', '666667777788889')");
+        conn.createStatement().execute("UPSERT INTO " + tableName
+                + " VALUES('000001111122222', '333334444455555', 0, 0, 'abc', '666667777788899')");
+        conn.commit();
+        QueryBuilder queryBuilder = new QueryBuilder()
+            .setSelectExpression("COUNT(1), EXTERNAL_DATASOURCE_KEY As DUP_COUNT")
+            .setSelectExpressionColumns(Lists.newArrayList("EXTERNAL_DATASOURCE_KEY", "MATCH_STATUS",
+                "JOURNEY_ID", "DATASOURCE", "ORGANIZATION_ID"))
+            .setWhereClause(
+            "JOURNEY_ID='333334444455555' AND DATASOURCE=0 AND MATCH_STATUS <= 1 and ORGANIZATION_ID='000001111122222'")
+            .setFullTableName(tableName)
+            .setGroupByClause("MATCH_STATUS, EXTERNAL_DATASOURCE_KEY")
+            .setHavingClause("COUNT(1) > 1");
+        ResultSet rs = executeQuery(conn, queryBuilder);
+        assertTrue(rs.next());
+        assertEquals(2,rs.getLong(1));
+        assertEquals("abc", rs.getString(2));
+        assertFalse(rs.next());
+        
+        String expectedPhoenixPlan = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName
+                + " ['000001111122222','333334444455555',0,*] - ['000001111122222','333334444455555',0,1]\n" +
+                "    SERVER FILTER BY FIRST KEY ONLY\n" +
+                "    SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [MATCH_STATUS, EXTERNAL_DATASOURCE_KEY]\n" +
+                "CLIENT FILTER BY COUNT(1) > 1";
+        validateQueryPlan(conn, queryBuilder, expectedPhoenixPlan, null);
+    }
+    
+    @Test
+    public void testGroupByOrderPreservingDescSort() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String tableName = generateUniqueName();
+        conn.createStatement().execute("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 char(1) not null," +
+                " constraint pk primary key (k1,k2)) split on ('ac','jc','nc')");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'a')");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'b')");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'c')");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'd')");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'a')");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'b')");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'c')");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'd')");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'a')");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'b')");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'c')");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'd')");
+        conn.commit();
+        QueryBuilder queryBuilder = new QueryBuilder()
+            .setSelectExpression("K1,COUNT(*)")
+            .setSelectColumns(Lists.newArrayList("K1"))
+            .setFullTableName(tableName)
+            .setGroupByClause("K1")
+            .setOrderByClause("K1 DESC");
+        ResultSet rs = executeQuery(conn, queryBuilder);
+        assertTrue(rs.next());
+        assertEquals("n", rs.getString(1));
+        assertEquals(4, rs.getLong(2));
+        assertTrue(rs.next());
+        assertEquals("j", rs.getString(1));
+        assertEquals(4, rs.getLong(2));
+        assertTrue(rs.next());
+        assertEquals("a", rs.getString(1));
+        assertEquals(4, rs.getLong(2));
+        assertFalse(rs.next());
+        String expectedPhoenixPlan = "CLIENT PARALLEL 1-WAY REVERSE FULL SCAN OVER " + tableName + "\n" +
+                "    SERVER FILTER BY FIRST KEY ONLY\n" +
+                "    SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]";
+        validateQueryPlan(conn, queryBuilder, expectedPhoenixPlan, null);
+    }
+    
+    @Test
+    public void testSumGroupByOrderPreservingDesc() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String tableName = generateUniqueName();
+
+        PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) split on (?,?,?)");
+        stmt.setBytes(1, ByteUtil.concat(PChar.INSTANCE.toBytes("a"), PInteger.INSTANCE.toBytes(3)));
+        stmt.setBytes(2, ByteUtil.concat(PChar.INSTANCE.toBytes("j"), PInteger.INSTANCE.toBytes(3)));
+        stmt.setBytes(3, ByteUtil.concat(PChar.INSTANCE.toBytes("n"), PInteger.INSTANCE.toBytes(3)));
+        stmt.execute();
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 1)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 2)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 3)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 4)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('b', 5)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 1)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 2)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 3)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 4)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 1)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 4)");
+        conn.commit();
+        QueryBuilder queryBuilder = new QueryBuilder()
+            .setSelectExpression("K1,SUM(K2)")
+            .setSelectExpressionColumns(Lists.newArrayList("K1", "K2"))
+            .setFullTableName(tableName)
+            .setGroupByClause("K1")
+            .setOrderByClause("K1 DESC");
+        ResultSet rs = executeQuery(conn, queryBuilder);
+        assertTrue(rs.next());
+        assertEquals("n", rs.getString(1));
+        assertEquals(10, rs.getLong(2));
+        assertTrue(rs.next());
+        assertEquals("j", rs.getString(1));
+        assertEquals(10, rs.getLong(2));
+        assertTrue(rs.next());
+        assertEquals("b", rs.getString(1));
+        assertEquals(5, rs.getLong(2));
+        assertTrue(rs.next());
+        assertEquals("a", rs.getString(1));
+        assertEquals(10, rs.getLong(2));
+        assertFalse(rs.next());
+        String expectedPhoenixPlan = "CLIENT PARALLEL 1-WAY REVERSE FULL SCAN OVER " + tableName + "\n" +
+                "    SERVER FILTER BY FIRST KEY ONLY\n" +
+                "    SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]";
+        validateQueryPlan(conn, queryBuilder, expectedPhoenixPlan, null);
+    }
+
+    @Test
+    public void testAvgGroupByOrderPreservingWithNoStats() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String tableName = generateUniqueName();
+        initAvgGroupTable(conn, tableName, "");
+        testAvgGroupByOrderPreserving(conn, tableName, 4);
+    }
+    
+    protected void initAvgGroupTable(Connection conn, String tableName, String tableProps) throws SQLException {
+        PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) " + tableProps + " split on (?,?,?)");
+        stmt.setBytes(1, ByteUtil.concat(PChar.INSTANCE.toBytes("a"), PInteger.INSTANCE.toBytes(3)));
+        stmt.setBytes(2, ByteUtil.concat(PChar.INSTANCE.toBytes("j"), PInteger.INSTANCE.toBytes(3)));
+        stmt.setBytes(3, ByteUtil.concat(PChar.INSTANCE.toBytes("n"), PInteger.INSTANCE.toBytes(3)));
+        stmt.execute();
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 1)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 2)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 3)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 6)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('b', 5)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 1)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 2)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 3)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 10)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 1)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
+        conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
+        conn.commit();
+    }
+    
+    protected void testAvgGroupByOrderPreserving(Connection conn, String tableName, int nGuidePosts) throws SQLException, IOException {
+        QueryBuilder queryBuilder = new QueryBuilder()
+            .setSelectExpression("K1, AVG(K2)")
+            .setSelectExpressionColumns(Lists.newArrayList("K1", "K2"))
+            .setFullTableName(tableName)
+            .setGroupByClause("K1")
+            .setOrderByClause("K1");
+        ResultSet rs = executeQuery(conn, queryBuilder);
+        assertTrue(rs.next());
+        assertEquals("a", rs.getString(1));
+        assertEquals(3, rs.getDouble(2), 1e-6);
+        assertTrue(rs.next());
+        assertEquals("b", rs.getString(1));
+        assertEquals(5, rs.getDouble(2), 1e-6);
+        assertTrue(rs.next());
+        assertEquals("j", rs.getString(1));
+        assertEquals(4, rs.getDouble(2), 1e-6);
+        assertTrue(rs.next());
+        assertEquals("n", rs.getString(1));
+        assertEquals(2, rs.getDouble(2), 1e-6);
+        assertFalse(rs.next());
+        String expectedPhoenixPlan = "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + tableName + "\n" +
+                "    SERVER FILTER BY FIRST KEY ONLY\n" +
+                "    SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]";
+        validateQueryPlan(conn, queryBuilder, expectedPhoenixPlan, null);
+        TestUtil.analyzeTable(conn, tableName);
+        List<KeyRange> splits = TestUtil.getAllSplits(conn, tableName);
+        assertEquals(nGuidePosts, splits.size());
+    }
+    
+    @Test
+    public void testDistinctGroupByBug3452WithoutMultiTenant() throws Exception {
+        doTestDistinctGroupByBug3452("");
+    }
+
+    @Test
+    public void testDistinctGroupByBug3452WithMultiTenant() throws Exception {
+        doTestDistinctGroupByBug3452("VERSIONS=1, MULTI_TENANT=TRUE, REPLICATION_SCOPE=1, TTL=31536000");
+    }
+
+    private void doTestDistinctGroupByBug3452(String options) throws Exception {
+        Connection conn=null;
+        try {
+            Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+            conn = DriverManager.getConnection(getUrl(), props);
+
+            String tableName=generateUniqueName();
+            conn.createStatement().execute("DROP TABLE if exists "+tableName);
+            String sql="CREATE TABLE "+ tableName +" ( "+
+                    "ORGANIZATION_ID CHAR(15) NOT NULL,"+
+                    "CONTAINER_ID CHAR(15) NOT NULL,"+
+                    "ENTITY_ID CHAR(15) NOT NULL,"+
+                    "SCORE DOUBLE,"+
+                    "CONSTRAINT TEST_PK PRIMARY KEY ( "+
+                    "ORGANIZATION_ID,"+
+                    "CONTAINER_ID,"+
+                    "ENTITY_ID"+
+                    ")) "+options;
+            conn.createStatement().execute(sql);
+
+            String indexTableName=generateUniqueName();
+            conn.createStatement().execute("DROP INDEX IF EXISTS "+indexTableName+" ON "+tableName);
+            conn.createStatement().execute("CREATE INDEX "+indexTableName+" ON "+tableName+" (CONTAINER_ID, SCORE DESC, ENTITY_ID DESC)");
+
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId6',1.1)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId5',1.2)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId4',1.3)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId3',1.4)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId2',1.5)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId1',1.6)");
+            conn.commit();
+
+            QueryBuilder queryBuilder = new QueryBuilder()
+                .setDistinct(true)
+                .setSelectColumns(Lists.newArrayList("ENTITY_ID", "SCORE", "ORGANIZATION_ID", "CONTAINER_ID"))
+                .setFullTableName(tableName)
+                .setWhereClause("ORGANIZATION_ID = 'org1' AND CONTAINER_ID = 'container1'")
+                .setOrderByClause("SCORE DESC");
+            ResultSet rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertTrue(rs.getString(1).equals("entityId1"));
+            assertEquals(rs.getDouble(2),1.6,0.0001);
+
+            assertTrue(rs.next());
+            assertTrue(rs.getString(1).equals("entityId2"));
+            assertEquals(rs.getDouble(2),1.5,0.0001);
+
+            assertTrue(rs.next());
+            assertTrue(rs.getString(1).equals("entityId3"));
+            assertEquals(rs.getDouble(2),1.4,0.0001);
+
+            assertTrue(rs.next());
+            assertTrue(rs.getString(1).equals("entityId4"));
+            assertEquals(rs.getDouble(2),1.3,0.0001);
+
+            assertTrue(rs.next());
+            assertTrue(rs.getString(1).equals("entityId5"));
+            assertEquals(rs.getDouble(2),1.2,0.0001);
+
+            assertTrue(rs.next());
+            assertTrue(rs.getString(1).equals("entityId6"));
+            assertEquals(rs.getDouble(2),1.1,0.0001);
+            assertTrue(!rs.next());
+        } finally {
+            if(conn!=null) {
+                conn.close();
+            }
+        }
+    }
+
+    @Test
+    public void testGroupByOrderByDescBug3451() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+            String tableName=generateUniqueName();
+            String sql="CREATE TABLE " + tableName + " (\n" + 
+                    "            ORGANIZATION_ID CHAR(15) NOT NULL,\n" + 
+                    "            CONTAINER_ID CHAR(15) NOT NULL,\n" + 
+                    "            ENTITY_ID CHAR(15) NOT NULL,\n" + 
+                    "            SCORE DOUBLE,\n" + 
+                    "            CONSTRAINT TEST_PK PRIMARY KEY (\n" + 
+                    "               ORGANIZATION_ID,\n" + 
+                    "               CONTAINER_ID,\n" + 
+                    "               ENTITY_ID\n" + 
+                    "             )\n" + 
+                    "         )";
+            conn.createStatement().execute(sql);
+            String indexName=generateUniqueName();
+            conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + "(ORGANIZATION_ID,CONTAINER_ID, SCORE DESC, ENTITY_ID DESC)");
+            conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container2','entityId6',1.1)");
+            conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container1','entityId5',1.2)");
+            conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container2','entityId4',1.3)");
+            conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container1','entityId5',1.2)");
+            conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container1','entityId3',1.4)");
+            conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container3','entityId7',1.35)");
+            conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container3','entityId8',1.45)");
+            conn.commit();
+            QueryBuilder queryBuilder = new QueryBuilder()
+                .setDistinct(true)
+                .setSelectColumns(Lists.newArrayList("ENTITY_ID", "SCORE", "ORGANIZATION_ID", "CONTAINER_ID"))
+                .setFullTableName(tableName)
+                .setWhereClause(
+                    "ORGANIZATION_ID = 'org2' AND CONTAINER_ID IN ('container1','container2','container3')")
+                .setOrderByClause("SCORE DESC")
+                .setLimit(2);
+            ResultSet rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertEquals("entityId8", rs.getString(1));
+            assertEquals(1.45, rs.getDouble(2),0.001);
+            assertTrue(rs.next());
+            assertEquals("entityId3", rs.getString(1));
+            assertEquals(1.4, rs.getDouble(2),0.001);
+            assertFalse(rs.next());
+
+            String expectedPhoenixPlan = "";
+            validateQueryPlan(conn, queryBuilder, expectedPhoenixPlan, null);
+       }
+    }
+    
+    @Test
+    public void testGroupByDescColumnWithNullsLastBug3452() throws Exception {
+
+        Connection conn=null;
+        try
+        {
+            Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+            conn = DriverManager.getConnection(getUrl(), props);
+
+            String tableName=generateUniqueName();
+            String sql="CREATE TABLE "+tableName+" ( "+
+                    "ORGANIZATION_ID VARCHAR,"+
+                    "CONTAINER_ID VARCHAR,"+
+                    "ENTITY_ID VARCHAR NOT NULL,"+
+                    "CONSTRAINT TEST_PK PRIMARY KEY ( "+
+                    "ORGANIZATION_ID DESC,"+
+                    "CONTAINER_ID DESC,"+
+                    "ENTITY_ID"+
+                    "))";
+            conn.createStatement().execute(sql);
+
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('a',null,'11')");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'2','22')");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('c','3','33')");
+            conn.commit();
+
+            //-----ORGANIZATION_ID
+
+            QueryBuilder queryBuilder = new QueryBuilder()
+                .setSelectColumns(
+                        Lists.newArrayList("CONTAINER_ID", "ORGANIZATION_ID"))
+                .setFullTableName(tableName)
+                .setGroupByClause("ORGANIZATION_ID, CONTAINER_ID")
+                .setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST");
+            ResultSet rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{null,"a"},{"3","c"},});
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null}});
+
+            //----CONTAINER_ID
+
+            queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{"2",null},{"3","c"}});
+
+            queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
+
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
+
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"}});
+
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,'44')");
+            conn.commit();
+
+            //-----ORGANIZATION_ID ASC  CONTAINER_ID ASC
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
+
+            //-----ORGANIZATION_ID ASC  CONTAINER_ID DESC
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
+
+            //-----ORGANIZATION_ID DESC  CONTAINER_ID ASC
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
+
+            //-----ORGANIZATION_ID DESC  CONTAINER_ID DESC
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
+
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
+
+            //-----CONTAINER_ID ASC  ORGANIZATION_ID ASC
+
+            queryBuilder.setOrderByClause("CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
+
+            queryBuilder.setOrderByClause("CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
+
+            queryBuilder.setOrderByClause("CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
+
+            queryBuilder.setOrderByClause("CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
+
+            //-----CONTAINER_ID ASC  ORGANIZATION_ID DESC
+
+            queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
+
+            queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
+
+            queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
+
+            queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
+
+            //-----CONTAINER_ID DESC  ORGANIZATION_ID ASC
+
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
+
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
+
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
+
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
+
+            //-----CONTAINER_ID DESC  ORGANIZATION_ID DESC
+
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
+
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
+
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
+
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
+        } finally {
+            if(conn!=null) {
+                conn.close();
+            }
+        }
+    }
+
+    @Test
+    public void testCountNullInEncodedNonEmptyKeyValueCF() throws Exception {
+        testCountNullInNonEmptyKeyValueCF(1);
+    }
+    
+    @Test
+    public void testCountNullInNonEncodedNonEmptyKeyValueCF() throws Exception {
+        testCountNullInNonEmptyKeyValueCF(0);
+    }
+
+    protected abstract void testCountNullInNonEmptyKeyValueCF(int columnEncodedBytes) throws Exception;
+
+    @Test
+    public void testGroupByOrderMatchPkColumnOrderBug4690() throws Exception {
+        this.doTestGroupByOrderMatchPkColumnOrderBug4690(false, false);
+        this.doTestGroupByOrderMatchPkColumnOrderBug4690(false, true);
+        this.doTestGroupByOrderMatchPkColumnOrderBug4690(true, false);
+        this.doTestGroupByOrderMatchPkColumnOrderBug4690(true, true);
+    }
+
+    private void doTestGroupByOrderMatchPkColumnOrderBug4690(boolean desc ,boolean salted) throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = null;
+        try {
+            conn = DriverManager.getConnection(getUrl(), props);
+            String tableName = generateUniqueName();
+            String sql = "create table " + tableName + "( "+
+                    " pk1 integer not null , " +
+                    " pk2 integer not null, " +
+                    " pk3 integer not null," +
+                    " pk4 integer not null,"+
+                    " v integer, " +
+                    " CONSTRAINT TEST_PK PRIMARY KEY ( "+
+                       "pk1 "+(desc ? "desc" : "")+", "+
+                       "pk2 "+(desc ? "desc" : "")+", "+
+                       "pk3 "+(desc ? "desc" : "")+", "+
+                       "pk4 "+(desc ? "desc" : "")+
+                    " )) "+(salted ? "SALT_BUCKETS =4" : "split on(2)");
+            conn.createStatement().execute(sql);
+
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,8,10,20,30)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,8,11,21,31)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,5 ,22,32)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,6 ,12,33)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,6 ,13,34)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,7 ,8,35)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,3,15,25,35)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,7,16,26,36)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,7,17,27,37)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,2,18,28,38)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,2,19,29,39)");
+            conn.commit();
+
+            QueryBuilder queryBuilder = new QueryBuilder()
+                .setSelectExpression("PK2,PK1,COUNT(V)")
+                .setSelectExpressionColumns(Lists.newArrayList("PK1", "PK2", "V"))
+                .setFullTableName(tableName)
+                .setGroupByClause("PK2, PK1")
+                .setOrderByClause("PK2, PK1");
+            ResultSet rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{2,3,2L},{3,2,1L},{7,2,2L},{8,1,2L},{9,1,4L}});
+
+            queryBuilder.setSelectExpression("PK1, PK2, COUNT(V)");
+            queryBuilder.setOrderByClause("PK1, PK2");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{1,8,2L},{1,9,4L},{2,3,1L},{2,7,2L},{3,2,2L}});
+
+            queryBuilder.setSelectExpression("PK2,PK1,COUNT(V)");
+            queryBuilder.setOrderByClause("PK2 DESC,PK1 DESC");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{9,1,4L},{8,1,2L},{7,2,2L},{3,2,1L},{2,3,2L}});
+
+            queryBuilder.setSelectExpression("PK1,PK2,COUNT(V)");
+            queryBuilder.setOrderByClause("PK1 DESC,PK2 DESC");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{3,2,2L},{2,7,2L},{2,3,1L},{1,9,4L},{1,8,2L}});
+
+
+            queryBuilder.setSelectExpression("PK3,PK2,COUNT(V)");
+            queryBuilder.setSelectExpressionColumns(Lists.newArrayList("PK1", "PK2", "PK3", "V"));
+            queryBuilder.setFullTableName(tableName);
+            queryBuilder.setGroupByClause("PK3,PK2");
+            queryBuilder.setOrderByClause("PK3,PK2");
+            queryBuilder.setWhereClause("PK1=1");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{5,9,1L},{6,9,2L},{7,9,1L},{10,8,1L},{11,8,1L}});
+
+            queryBuilder.setSelectExpression("PK2,PK3,COUNT(V)");
+            queryBuilder.setOrderByClause("PK2,PK3");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{8,10,1L},{8,11,1L},{9,5,1L},{9,6,2L},{9,7,1L}});
+
+            queryBuilder.setSelectExpression("PK3,PK2,COUNT(V)");
+            queryBuilder.setOrderByClause("PK3 DESC,PK2 DESC");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{11,8,1L},{10,8,1L},{7,9,1L},{6,9,2L},{5,9,1L}});
+
+            queryBuilder.setSelectExpression("PK2,PK3,COUNT(V)");
+            queryBuilder.setOrderByClause("PK2 DESC,PK3 DESC");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{9,7,1L},{9,6,2L},{9,5,1L},{8,11,1L},{8,10,1L}});
+
+
+            queryBuilder.setSelectExpression("PK4,PK3,PK1,COUNT(V)");
+            queryBuilder.setSelectExpressionColumns(Lists.newArrayList("PK1", "PK2", "PK3", "PK4", "V"));
+            queryBuilder.setFullTableName(tableName);
+            queryBuilder.setWhereClause("PK2=9 ");
+            queryBuilder.setGroupByClause("PK4,PK3,PK1");
+            queryBuilder.setOrderByClause("PK4,PK3,PK1");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{8,7,1,1L},{12,6,1,1L},{13,6,1,1L},{22,5,1,1L}});
+
+            queryBuilder.setSelectExpression("PK1,PK3,PK4,COUNT(V)");
+            queryBuilder.setOrderByClause("PK1,PK3,PK4");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{1,5,22,1L},{1,6,12,1L},{1,6,13,1L},{1,7,8,1L}});
+
+            queryBuilder.setSelectExpression("PK4,PK3,PK1,COUNT(V)");
+            queryBuilder.setOrderByClause("PK4 DESC,PK3 DESC,PK1 DESC");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{22,5,1,1L},{13,6,1,1L},{12,6,1,1L},{8,7,1,1L}});
+
+            queryBuilder.setSelectExpression("PK1,PK3,PK4,COUNT(V)");
+            queryBuilder.setOrderByClause("PK1 DESC,PK3 DESC,PK4 DESC");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{1,7,8,1L},{1,6,13,1L},{1,6,12,1L},{1,5,22,1L}});
+        } finally {
+            if(conn != null) {
+                conn.close();
+            }
+        }
+    }
+}


[22/22] phoenix git commit: Remove unnecessary dependencies in assembly

Posted by ja...@apache.org.
Remove unnecessary dependencies in assembly


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d03eadc2
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d03eadc2
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d03eadc2

Branch: refs/heads/omid2
Commit: d03eadc23f7da4bcfa36c7add810ff43707e5700
Parents: cf08715
Author: James Taylor <ja...@apache.org>
Authored: Sat Nov 10 12:56:59 2018 -0800
Committer: James Taylor <ja...@apache.org>
Committed: Sat Nov 10 12:58:32 2018 -0800

----------------------------------------------------------------------
 phoenix-assembly/pom.xml | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/d03eadc2/phoenix-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/pom.xml b/phoenix-assembly/pom.xml
index 9d3c891..e2543c5 100644
--- a/phoenix-assembly/pom.xml
+++ b/phoenix-assembly/pom.xml
@@ -127,23 +127,13 @@
       <artifactId>phoenix-queryserver-client</artifactId>
     </dependency>
     <dependency>
-      <groupId>com.google.inject</groupId>
-      <artifactId>guice</artifactId>
-      <version>3.0</version>
+      <groupId>org.apache.phoenix</groupId>
+      <artifactId>phoenix-load-balancer</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.omid</groupId>
       <artifactId>omid-hbase-tools-hbase1.x</artifactId>
       <version>${omid.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.apache.htrace</groupId>
-      <artifactId>htrace-core</artifactId>
-      <version>3.2.0-incubating</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.phoenix</groupId>
-      <artifactId>phoenix-load-balancer</artifactId>
-    </dependency>
   </dependencies>
 </project>


[13/22] phoenix git commit: PHOENIX-4981 Add tests for ORDER BY, GROUP BY and salted tables using phoenix-spark

Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByIT.java
new file mode 100644
index 0000000..31bf050
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByIT.java
@@ -0,0 +1,940 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.apache.phoenix.util.TestUtil.ROW1;
+import static org.apache.phoenix.util.TestUtil.ROW2;
+import static org.apache.phoenix.util.TestUtil.ROW3;
+import static org.apache.phoenix.util.TestUtil.ROW4;
+import static org.apache.phoenix.util.TestUtil.ROW5;
+import static org.apache.phoenix.util.TestUtil.ROW6;
+import static org.apache.phoenix.util.TestUtil.ROW7;
+import static org.apache.phoenix.util.TestUtil.ROW8;
+import static org.apache.phoenix.util.TestUtil.ROW9;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.apache.phoenix.util.TestUtil.assertResultSet;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.Properties;
+
+import com.google.common.collect.Lists;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryBuilder;
+import org.junit.Test;
+
+
+public abstract class BaseOrderByIT extends ParallelStatsDisabledIT {
+
+    @Test
+    public void testMultiOrderByExpr() throws Exception {
+        String tenantId = getOrganizationId();
+        String tableName = initATableValues(tenantId, getDefaultSplits(tenantId), getUrl());
+        QueryBuilder queryBuilder = new QueryBuilder()
+            .setSelectColumns(
+                    Lists.newArrayList("ENTITY_ID", "B_STRING"))
+            .setFullTableName(tableName)
+            .setOrderByClause("B_STRING, ENTITY_ID");
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+            ResultSet rs = executeQuery(conn, queryBuilder);
+            assertTrue (rs.next());
+            assertEquals(ROW1,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW4,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW7,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW2,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW5,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW8,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW3,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW6,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW9,rs.getString(1));
+
+            assertFalse(rs.next());
+        }
+    }
+
+
+    @Test
+    public void testDescMultiOrderByExpr() throws Exception {
+        String tenantId = getOrganizationId();
+        String tableName = initATableValues(tenantId, getDefaultSplits(tenantId), getUrl());
+        QueryBuilder queryBuilder = new QueryBuilder()
+            .setSelectColumns(
+                    Lists.newArrayList("ENTITY_ID", "B_STRING"))
+            .setFullTableName(tableName)
+            .setOrderByClause("B_STRING || ENTITY_ID DESC");
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+            ResultSet rs = executeQuery(conn, queryBuilder);
+            assertTrue (rs.next());
+            assertEquals(ROW9,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW6,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW3,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW8,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW5,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW2,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW7,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW4,rs.getString(1));
+            assertTrue (rs.next());
+            assertEquals(ROW1,rs.getString(1));
+
+            assertFalse(rs.next());
+        }
+    }
+
+    @Test
+    public void testOrderByDifferentColumns() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+            conn.setAutoCommit(false);
+            String tableName = generateUniqueName();
+            String ddl = "CREATE TABLE " + tableName +
+                    "  (a_string varchar not null, col1 integer" +
+                    "  CONSTRAINT pk PRIMARY KEY (a_string))\n";
+            createTestTable(getUrl(), ddl);
+
+            String dml = "UPSERT INTO " + tableName + " VALUES(?, ?)";
+            PreparedStatement stmt = conn.prepareStatement(dml);
+            stmt.setString(1, "a");
+            stmt.setInt(2, 40);
+            stmt.execute();
+            stmt.setString(1, "b");
+            stmt.setInt(2, 20);
+            stmt.execute();
+            stmt.setString(1, "c");
+            stmt.setInt(2, 30);
+            stmt.execute();
+            conn.commit();
+
+            QueryBuilder queryBuilder = new QueryBuilder()
+                .setSelectColumns(
+                        Lists.newArrayList("COL1"))
+                .setFullTableName(tableName)
+                .setSelectExpression("count(*)")
+                .setGroupByClause("COL1")
+                .setOrderByClause("COL1");
+            ResultSet rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertEquals(1,rs.getLong(1));
+            assertTrue(rs.next());
+            assertEquals(1,rs.getLong(1));
+            assertTrue(rs.next());
+            assertEquals(1,rs.getLong(1));
+            assertFalse(rs.next());
+
+            queryBuilder = new QueryBuilder();
+            queryBuilder.setSelectColumns(
+                    Lists.newArrayList("A_STRING",  "COL1"));
+            queryBuilder.setFullTableName(tableName);
+            queryBuilder.setOrderByClause("A_STRING");
+            rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertEquals("a",rs.getString(1));
+            assertEquals(40,rs.getInt(2));
+            assertTrue(rs.next());
+            assertEquals("b",rs.getString(1));
+            assertEquals(20,rs.getInt(2));
+            assertTrue(rs.next());
+            assertEquals("c",rs.getString(1));  
+            assertEquals(30,rs.getInt(2));
+            assertFalse(rs.next());  
+
+            queryBuilder.setSelectColumns(
+                    Lists.newArrayList("A_STRING", "COL1"));
+            queryBuilder.setFullTableName(tableName);
+            queryBuilder.setOrderByClause("COL1");
+            rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertEquals("b",rs.getString(1));
+            assertEquals(20,rs.getInt(2));
+            assertTrue(rs.next());
+            assertEquals("c",rs.getString(1));  
+            assertEquals(30,rs.getInt(2));
+            assertTrue(rs.next());
+            assertEquals("a",rs.getString(1));
+            assertEquals(40,rs.getInt(2));
+            assertFalse(rs.next());  
+        }
+    }
+
+    @Test
+    public void testAggregateOrderBy() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String tableName = generateUniqueName();
+        String ddl = "create table " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, VAL1 VARCHAR, VAL2 INTEGER)";
+        conn.createStatement().execute(ddl);
+
+        conn.createStatement().execute("upsert into " + tableName + " values ('ABC','aa123', 11)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('ABD','ba124', 1)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('ABE','cf125', 13)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('ABF','dan126', 4)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('ABG','elf127', 15)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('ABH','fan128', 6)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('AAA','get211', 100)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('AAB','hat212', 7)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('AAC','aap12', 2)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('AAD','ball12', 3)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('AAE','inn2110', 13)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('AAF','key2112', 40)");
+        conn.commit();
+
+        QueryBuilder queryBuilder = new QueryBuilder()
+            .setSelectColumns(
+                    Lists.newArrayList("VAL1", "VAL2"))
+            .setFullTableName(tableName)
+            .setOrderByClause("VAL1")
+            .setSelectExpression("DISTINCT(ID)")
+            .setSelectExpressionColumns(Lists.newArrayList("ID"))
+            .setWhereClause(
+                "ID in ('ABC','ABD','ABE','ABF','ABG','ABH','AAA', 'AAB', 'AAC','AAD','AAE','AAF')");
+        ResultSet rs = executeQuery(conn, queryBuilder);
+        assertTrue(rs.next());
+        assertEquals("ABC", rs.getString(1));
+        assertEquals("aa123", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("aap12", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("ba124", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("ball12", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("cf125", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("dan126", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("elf127", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("fan128", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("get211", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("hat212", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("inn2110", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("AAF", rs.getString(1));
+        assertEquals("key2112", rs.getString(2));
+        assertFalse(rs.next());
+    }
+
+    @Test
+    public void testAggregateOptimizedOutOrderBy() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String tableName = generateUniqueName();
+        String ddl = "create table " + tableName + " (K1 VARCHAR NOT NULL, K2 VARCHAR NOT NULL, VAL1 VARCHAR, VAL2 INTEGER, CONSTRAINT pk PRIMARY KEY(K1,K2))";
+        conn.createStatement().execute(ddl);
+
+        conn.createStatement().execute("upsert into " + tableName + " values ('ABC','ABC','aa123', 11)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('ABD','ABC','ba124', 1)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('ABE','ABC','cf125', 13)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('ABF','ABC','dan126', 4)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('ABG','ABC','elf127', 15)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('ABH','ABC','fan128', 6)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('AAA','ABC','get211', 100)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('AAB','ABC','hat212', 7)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('AAC','ABC','aap12', 2)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('AAD','ABC','ball12', 3)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('AAE','ABC','inn2110', 13)");
+        conn.createStatement().execute("upsert into " + tableName + " values ('AAF','ABC','key2112', 40)");
+        conn.commit();
+
+        QueryBuilder queryBuilder = new QueryBuilder()
+            .setSelectColumns(
+                    Lists.newArrayList("VAL1", "VAL2"))
+            .setFullTableName(tableName)
+            .setOrderByClause("VAL1")
+            .setSelectExpressionColumns(Lists.newArrayList("K2"))
+            .setSelectExpression("DISTINCT(K2)")
+        .setWhereClause("K2 = 'ABC'");
+
+        // verify that the phoenix query plan doesn't contain an order by
+        String expectedPhoenixPlan = "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + tableName +"\n" +
+                "    SERVER FILTER BY K2 = 'ABC'\n" +
+                "    SERVER AGGREGATE INTO DISTINCT ROWS BY [K2, VAL1, VAL2]\n" +
+                "CLIENT MERGE SORT";
+        validateQueryPlan(conn, queryBuilder, expectedPhoenixPlan, null);
+
+        ResultSet rs = executeQuery(conn, queryBuilder);
+        assertTrue(rs.next());
+        assertEquals("ABC", rs.getString(1));
+        assertEquals("aa123", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("aap12", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("ba124", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("ball12", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("cf125", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("dan126", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("elf127", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("fan128", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("get211", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("hat212", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("inn2110", rs.getString(2));
+        assertTrue(rs.next());
+        assertEquals("ABC", rs.getString(1));
+        assertEquals("key2112", rs.getString(2));
+        assertFalse(rs.next());
+    }
+
+    @Test
+    public void testNullsLastWithDesc() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+            String tableName=generateUniqueName();
+            String sql="CREATE TABLE "+tableName+" ( "+
+                "ORGANIZATION_ID VARCHAR,"+
+                "CONTAINER_ID VARCHAR,"+
+                "ENTITY_ID VARCHAR NOT NULL,"+
+                "CONSTRAINT TEST_PK PRIMARY KEY ( "+
+                "ORGANIZATION_ID DESC,"+
+                "CONTAINER_ID DESC,"+
+                "ENTITY_ID"+
+                "))";
+            conn.createStatement().execute(sql);
+
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('a',null,'11')");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'2','22')");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('c','3','33')");
+            conn.commit();
+
+            //-----ORGANIZATION_ID
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID ASC NULLS FIRST";
+            QueryBuilder queryBuilder = new QueryBuilder()
+                .setSelectColumns(
+                        Lists.newArrayList("CONTAINER_ID", "ORGANIZATION_ID"))
+                .setFullTableName(tableName)
+                .setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST");
+            ResultSet rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{null,"a"},{"3","c"},});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID ASC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null}});
+
+            //----CONTAINER_ID
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS FIRST";
+            queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{"2",null},{"3","c"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS LAST";
+            queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST";
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"}});
+
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,'44')");
+            conn.commit();
+
+            //-----ORGANIZATION_ID ASC  CONTAINER_ID ASC
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
+
+
+            //-----ORGANIZATION_ID ASC  CONTAINER_ID DESC
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
+
+            //-----ORGANIZATION_ID DESC  CONTAINER_ID ASC
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
+
+            //-----ORGANIZATION_ID DESC  CONTAINER_ID DESC
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
+
+            //-----CONTAINER_ID ASC  ORGANIZATION_ID ASC
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS FIRST";
+            queryBuilder.setOrderByClause("CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS LAST";
+            queryBuilder.setOrderByClause("CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS FIRST";
+            queryBuilder.setOrderByClause("CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS LAST";
+            queryBuilder.setOrderByClause("CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
+
+            //-----CONTAINER_ID ASC  ORGANIZATION_ID DESC
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST";
+            queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST";
+            queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
+
+            //-----CONTAINER_ID DESC  ORGANIZATION_ID ASC
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS FIRST";
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS LAST";
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS FIRST";
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS LAST";
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
+
+            //-----CONTAINER_ID DESC  ORGANIZATION_ID DESC
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST";
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
+
+//            sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST";
+            queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
+        }
+    }
+
+    @Test
+    public void testOrderByReverseOptimization() throws Exception {
+        for(boolean salted: new boolean[]{true,false}) {
+            doTestOrderByReverseOptimization(salted,true,true,true);
+            doTestOrderByReverseOptimization(salted,true,true,false);
+            doTestOrderByReverseOptimization(salted,true,false,true);
+            doTestOrderByReverseOptimization(salted,true,false,false);
+            doTestOrderByReverseOptimization(salted,false,true,true);
+            doTestOrderByReverseOptimization(salted,false,true,false);
+            doTestOrderByReverseOptimization(salted,false,false,true);
+            doTestOrderByReverseOptimization(salted,false,false,false);
+        }
+    }
+
+    private void doTestOrderByReverseOptimization(boolean salted,boolean desc1,boolean desc2,boolean desc3) throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+            String tableName=generateUniqueName();
+            String sql="CREATE TABLE "+tableName+" ( "+
+                    "ORGANIZATION_ID INTEGER NOT NULL,"+
+                    "CONTAINER_ID INTEGER NOT NULL,"+
+                    "SCORE INTEGER NOT NULL,"+
+                    "ENTITY_ID INTEGER NOT NULL,"+
+                    "CONSTRAINT TEST_PK PRIMARY KEY ( "+
+                    "ORGANIZATION_ID" +(desc1 ? " DESC" : "" )+","+
+                    "CONTAINER_ID"+(desc2 ? " DESC" : "" )+","+
+                    "SCORE"+(desc3 ? " DESC" : "" )+","+
+                    "ENTITY_ID"+
+                    ")) "+(salted ? "SALT_BUCKETS =4" : "split on(4)");
+            conn.createStatement().execute(sql);
+
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,1,1,1)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,2,2,2)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,3,3,3)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (4,4,4,4)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (5,5,5,5)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (6,6,6,6)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,1,1,11)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,2,2,22)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,3,3,33)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (4,4,4,44)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (5,5,5,55)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (6,6,6,66)");
+            conn.commit();
+
+            QueryBuilder queryBuilder = new QueryBuilder()
+                .setSelectColumns(
+                        Lists.newArrayList("CONTAINER_ID", "ORGANIZATION_ID"))
+                .setFullTableName(tableName)
+                .setGroupByClause("ORGANIZATION_ID, CONTAINER_ID")
+            .setOrderByClause(  "ORGANIZATION_ID ASC, CONTAINER_ID ASC");
+            //groupBy orderPreserving orderBy asc asc
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC, CONTAINER_ID ASC";
+            ResultSet rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
+
+            //groupBy orderPreserving orderBy asc desc
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC, CONTAINER_ID desc";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC, CONTAINER_ID DESC");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
+
+            //groupBy orderPreserving orderBy desc asc
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC, CONTAINER_ID ASC";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC, CONTAINER_ID ASC");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
+
+            //groupBy orderPreserving orderBy desc desc
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC, CONTAINER_ID DESC";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC, CONTAINER_ID DESC");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
+
+            //groupBy not orderPreserving orderBy asc asc
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC, SCORE ASC";
+            queryBuilder.setSelectColumns(
+                    Lists.newArrayList("ORGANIZATION_ID", "SCORE"));
+            queryBuilder.setGroupByClause("ORGANIZATION_ID, SCORE");
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC, SCORE ASC");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
+
+            //groupBy not orderPreserving orderBy asc desc
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC, SCORE DESC";
+            queryBuilder.setGroupByClause("ORGANIZATION_ID, SCORE");
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC, SCORE DESC");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
+
+            //groupBy not orderPreserving orderBy desc asc
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC, SCORE ASC";
+            queryBuilder.setGroupByClause("ORGANIZATION_ID, SCORE");
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC, SCORE ASC");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
+
+            //groupBy not orderPreserving orderBy desc desc
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC, SCORE DESC";
+            queryBuilder.setGroupByClause("ORGANIZATION_ID, SCORE");
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC, SCORE DESC");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
+        }
+    }
+
+    @Test
+    public void testOrderByReverseOptimizationWithNullsLast() throws Exception{
+        for(boolean salted: new boolean[]{true,false}) {
+            doTestOrderByReverseOptimizationWithNullsLast(salted,true,true,true);
+            doTestOrderByReverseOptimizationWithNullsLast(salted,true,true,false);
+            doTestOrderByReverseOptimizationWithNullsLast(salted,true,false,true);
+            doTestOrderByReverseOptimizationWithNullsLast(salted,true,false,false);
+            doTestOrderByReverseOptimizationWithNullsLast(salted,false,true,true);
+            doTestOrderByReverseOptimizationWithNullsLast(salted,false,true,false);
+            doTestOrderByReverseOptimizationWithNullsLast(salted,false,false,true);
+            doTestOrderByReverseOptimizationWithNullsLast(salted,false,false,false);
+        }
+    }
+
+    private void doTestOrderByReverseOptimizationWithNullsLast(boolean salted,boolean desc1,boolean desc2,boolean desc3) throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+            String tableName=generateUniqueName();
+            String sql="CREATE TABLE "+tableName+" ( "+
+                    "ORGANIZATION_ID VARCHAR,"+
+                    "CONTAINER_ID VARCHAR,"+
+                    "SCORE VARCHAR,"+
+                    "ENTITY_ID VARCHAR NOT NULL,"+
+                    "CONSTRAINT TEST_PK PRIMARY KEY ( "+
+                    "ORGANIZATION_ID" +(desc1 ? " DESC" : "" )+","+
+                    "CONTAINER_ID"+(desc2 ? " DESC" : "" )+","+
+                    "SCORE"+(desc3 ? " DESC" : "" )+","+
+                    "ENTITY_ID"+
+                    ")) "+(salted ? "SALT_BUCKETS =4" : "split on('4')");
+            conn.createStatement().execute(sql);
+
+            for(int i=1;i<=6;i++) {
+                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'"+i+"','"+i+"','"+i+"')");
+                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'"+i+"',null,'"+i+"')");
+                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,'"+i+"','"+i+"')");
+                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,null,'"+i+"')");
+                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"','"+i+"','"+i+"','"+i+"')");
+                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"','"+i+"',null,'"+i+"')");
+                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"',null,'"+i+"','"+i+"')");
+                conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"',null,null,'"+i+"')");
+            }
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,null,'66')");
+            conn.commit();
+
+            //groupBy orderPreserving orderBy asc asc
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST";
+            QueryBuilder queryBuilder = new QueryBuilder()
+            .setSelectColumns(
+                    Lists.newArrayList("ORGANIZATION_ID", "CONTAINER_ID"))
+            .setFullTableName(tableName)
+            .setGroupByClause("ORGANIZATION_ID, CONTAINER_ID")
+            .setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST");
+            ResultSet rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID ASC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
+
+            //groupBy orderPreserving orderBy asc desc
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
+
+            //groupBy orderPreserving orderBy desc asc
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"}});
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
+
+            //groupBy orderPreserving orderBy desc desc
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
+
+//            sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
+
+            //-----groupBy not orderPreserving--
+
+            //groupBy not orderPreserving orderBy asc asc
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS FIRST";
+            queryBuilder.setSelectColumns(
+                    Lists.newArrayList("ORGANIZATION_ID", "SCORE"))
+                .setFullTableName(tableName)
+                .setGroupByClause("ORGANIZATION_ID, SCORE")
+                .setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"},{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"}});
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
+
+            //groupBy not orderPreserving orderBy asc desc
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
+
+            //groupBy not orderPreserving orderBy desc asc
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"}});
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
+
+            //groupBy not orderPreserving orderBy desc desc
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
+
+//            sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS LAST";
+            queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
+
+            //-------test only one return column----------------------------------
+
+//            sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE ASC NULLS FIRST";
+            queryBuilder.setSelectColumns(
+                    Lists.newArrayList("SCORE"))
+                .setFullTableName(tableName)
+                .setGroupByClause("SCORE")
+                .setOrderByClause("SCORE ASC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null},{"1"},{"2"},{"3"},{"4"},{"5"},{"6"}});
+
+//            sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE ASC NULLS LAST";
+            queryBuilder.setOrderByClause("SCORE ASC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"1"},{"2"},{"3"},{"4"},{"5"},{"6"},{null}});
+
+//            sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE DESC NULLS FIRST";
+            queryBuilder.setOrderByClause("SCORE DESC NULLS FIRST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{null},{"6"},{"5"},{"4"},{"3"},{"2"},{"1"}});
+
+//            sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE DESC NULLS LAST";
+            queryBuilder.setOrderByClause("SCORE DESC NULLS LAST");
+            rs = executeQuery(conn, queryBuilder);
+            assertResultSet(rs, new Object[][]{{"6"},{"5"},{"4"},{"3"},{"2"},{"1"},{null}});
+        }
+    }
+
+}
\ No newline at end of file


[05/22] phoenix git commit: PHOENIX-4891: An index should inherit UPDATE_CACHE_FREQUENCY setting rom parent table

Posted by ja...@apache.org.
PHOENIX-4891: An index should inherit UPDATE_CACHE_FREQUENCY setting rom parent table


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/64788794
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/64788794
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/64788794

Branch: refs/heads/omid2
Commit: 647887947f631d624d6897d729bce3acb07a94bf
Parents: fe466eb
Author: Chinmay Kulkarni <ch...@gmail.com>
Authored: Tue Oct 30 14:52:26 2018 -0700
Committer: Thomas D'Silva <td...@apache.org>
Committed: Tue Oct 30 16:23:42 2018 -0700

----------------------------------------------------------------------
 .../phoenix/end2end/PropertiesInSyncIT.java     | 172 ++++++++++++++-----
 .../phoenix/end2end/index/IndexMetadataIT.java  | 145 +++++++++++++++-
 .../org/apache/phoenix/rpc/UpdateCacheIT.java   | 134 ++++++++++-----
 .../phoenix/exception/SQLExceptionCode.java     |   7 +-
 .../query/ConnectionQueryServicesImpl.java      |   6 +-
 .../apache/phoenix/schema/MetaDataClient.java   |  33 +++-
 .../org/apache/phoenix/util/MetaDataUtil.java   |   6 +-
 .../org/apache/phoenix/util/UpgradeUtil.java    |  79 ++++++++-
 8 files changed, 476 insertions(+), 106 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
index db44735..348b195 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
@@ -23,9 +23,12 @@ import org.apache.hadoop.hbase.KeepDeletedCells;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
 import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.util.MetaDataUtil;
 import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.SchemaUtil;
@@ -33,17 +36,23 @@ import org.junit.Test;
 
 import java.sql.Connection;
 import java.sql.DriverManager;
+import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES;
-import static org.apache.phoenix.util.MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES;
+import static org.apache.phoenix.util.MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES;
 import static org.apache.phoenix.util.MetaDataUtil.VIEW_INDEX_TABLE_PREFIX;
+import static org.apache.phoenix.util.UpgradeUtil.UPSERT_UPDATE_CACHE_FREQUENCY;
 import static org.apache.phoenix.util.UpgradeUtil.syncTableAndIndexProperties;
+import static org.apache.phoenix.util.UpgradeUtil.syncUpdateCacheFreqAllIndexes;
+import static org.apache.phoenix.end2end.index.IndexMetadataIT.assertUpdateCacheFreq;
 
 /**
  * Test properties that need to be kept in sync amongst all column families and indexes of a table
@@ -56,12 +65,16 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
     private static final int INITIAL_TTL_VALUE = 700;
     private static final KeepDeletedCells INITIAL_KEEP_DELETED_CELLS_VALUE = KeepDeletedCells.TRUE;
     private static final int INITIAL_REPLICATION_SCOPE_VALUE = 1;
+    private static final int INITIAL_UPDATE_CACHE_FREQUENCY = 100;
+    private static final int INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS = 900;
     private static final int MODIFIED_TTL_VALUE = INITIAL_TTL_VALUE + 300;
     private static final KeepDeletedCells MODIFIED_KEEP_DELETED_CELLS_VALUE =
-            (INITIAL_KEEP_DELETED_CELLS_VALUE == KeepDeletedCells.TRUE)
-                    ? KeepDeletedCells.FALSE: KeepDeletedCells.TRUE;
+            (INITIAL_KEEP_DELETED_CELLS_VALUE == KeepDeletedCells.TRUE) ?
+                    KeepDeletedCells.FALSE: KeepDeletedCells.TRUE;
     private static final int MODIFIED_REPLICATION_SCOPE_VALUE =
             (INITIAL_REPLICATION_SCOPE_VALUE == 1) ? 0 : 1;
+    private static final int MODIFIED_UPDATE_CACHE_FREQUENCY = INITIAL_UPDATE_CACHE_FREQUENCY + 300;
+    private static final int MODIFIED_UPDATE_CACHE_FREQUENCY_VIEWS = INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS + 300;
 
 
     // Test that we disallow specifying synced properties to be set per column family
@@ -70,7 +83,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
     public void testDisallowSyncedPropsToBeSetColFamSpecificCreateTable() throws Exception {
         Connection conn = DriverManager.getConnection(getUrl(), new Properties());
         String tableName = generateUniqueName();
-        for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+        for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
             try {
                 conn.createStatement().execute("create table " + tableName
                         + " (id INTEGER not null primary key, "
@@ -107,7 +120,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
         String viewName = "VIEW_" + tableName;
         conn.createStatement().execute("create view " + viewName
                 + " (new_col SMALLINT) as select * from " + tableName + " where id > 1");
-        for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+        for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
             try {
                 conn.createStatement().execute("create local index " + localIndexName
                         + " on " + tableName + "(name) "
@@ -148,8 +161,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
     public void testSyncedPropsBaseTableCreateIndex() throws Exception {
         Connection conn = DriverManager.getConnection(getUrl(), new Properties());
         String tableName = createBaseTableWithProps(conn);
-        createIndexTable(conn, tableName, PTable.IndexType.LOCAL);
-        String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL);
+        createIndexTable(conn, tableName, PTable.IndexType.LOCAL).getSecond();
+        String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond();
 
         // We pass the base table as the physical HBase table since our check includes checking
         // the local index column family too
@@ -164,7 +177,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
     public void testSyncedPropsBaseTableCreateViewIndex() throws Exception {
         Connection conn = DriverManager.getConnection(getUrl(), new Properties());
         String tableName = createBaseTableWithProps(conn);
-        String viewIndexName = createIndexTable(conn, tableName, null);
+        String viewIndexName = createIndexTable(conn, tableName, null).getSecond();
 
         verifyHBaseColumnFamilyProperties(tableName, conn, false, false);
         verifyHBaseColumnFamilyProperties(viewIndexName, conn, false, false);
@@ -179,7 +192,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
         String tableName = createBaseTableWithProps(conn);
         StringBuilder alterAllSyncedPropsString = new StringBuilder();
         String modPropString = COL_FAM1 + ".%s=" + DUMMY_PROP_VALUE + ",";
-        for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+        for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
             try {
                 conn.createStatement().execute("alter table " + tableName
                         + " set " + COL_FAM1 + "." + propName + "=" + DUMMY_PROP_VALUE);
@@ -216,11 +229,11 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
         Set<String> tablesToCheck = new HashSet<>();
         tablesToCheck.add(tableName);
         for (int i=0; i<2; i++) {
-            tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL));
-            tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL));
+            tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL).getSecond());
+            tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond());
         }
         // Create a view and view index
-        tablesToCheck.add(createIndexTable(conn, tableName, null));
+        tablesToCheck.add(createIndexTable(conn, tableName, null).getSecond());
 
         // Now alter the base table's properties. This should get propagated to all index tables
         conn.createStatement().execute("alter table " + tableName + " set TTL=" + MODIFIED_TTL_VALUE
@@ -232,8 +245,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
         }
 
         // Any indexes created henceforth should have the modified properties
-        String newGlobalIndex = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL);
-        String newViewIndex = createIndexTable(conn, tableName, null);
+        String newGlobalIndex = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond();
+        String newViewIndex = createIndexTable(conn, tableName, null).getSecond();
         verifyHBaseColumnFamilyProperties(newGlobalIndex, conn, true, false);
         verifyHBaseColumnFamilyProperties(newViewIndex, conn, true, false);
         conn.close();
@@ -245,9 +258,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
         Connection conn = DriverManager.getConnection(getUrl(), new Properties());
         String tableName = createBaseTableWithProps(conn);
 
-        // Test that we are not allowed to set any property to be kept in sync, specific
-        // to the new column family to be added
-        for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+        // Test that we are not allowed to set any property to be kept in sync, specific to the new column family to be added
+        for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
             try {
                 conn.createStatement().execute(
                         "alter table " + tableName + " add " + NEW_CF + ".new_column varchar(2) "
@@ -268,11 +280,11 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
         Set<String> tablesToCheck = new HashSet<>();
         tablesToCheck.add(tableName);
         for (int i=0; i<2; i++) {
-            tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL));
-            tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL));
+            tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL).getSecond());
+            tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond());
         }
         // Create a view and view index
-        tablesToCheck.add(createIndexTable(conn, tableName, null));
+        tablesToCheck.add(createIndexTable(conn, tableName, null).getSecond());
 
         // Now add a new column family while simultaneously modifying properties to be kept in sync,
         // as well as a property which does not need to be kept in sync. Properties to be kept
@@ -307,8 +319,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
     public void testDisallowAlterGlobalIndexTable() throws Exception {
         Connection conn = DriverManager.getConnection(getUrl(), new Properties());
         String tableName = createBaseTableWithProps(conn);
-        String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL);
-        for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+        String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond();
+        for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
             try {
                 conn.createStatement().execute("alter table " + globalIndexName + " set "
                 + propName + "=" + DUMMY_PROP_VALUE);
@@ -336,12 +348,12 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
         createdTables.add(baseTableName1);
         // Create different indexes on the base table
         for (int i=0; i<2; i++) {
-            createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.GLOBAL));
-            createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.LOCAL));
-            createdTables.add(createIndexTable(conn, baseTableName, null));
-            createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.GLOBAL));
-            createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.LOCAL));
-            createdTables.add(createIndexTable(conn, baseTableName1, null));
+            createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.GLOBAL).getSecond());
+            createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.LOCAL).getSecond());
+            createdTables.add(createIndexTable(conn, baseTableName, null).getSecond());
+            createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.GLOBAL).getSecond());
+            createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.LOCAL).getSecond());
+            createdTables.add(createIndexTable(conn, baseTableName1, null).getSecond());
         }
         for (String t: createdTables) {
             verifyHBaseColumnFamilyProperties(t, conn, false, false);
@@ -394,6 +406,77 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
         conn.close();
     }
 
+    @Test
+    public void testOldClientSyncUpdateCacheFreqUpgradePath() throws Exception {
+        PTable base, index;
+        String baseTableName, viewName, viewName2;
+        Map<String, Set<String>> createdTablesAndViews = new HashMap<>();
+
+        try (Connection conn = DriverManager.getConnection(getUrl(), new Properties())) {
+            baseTableName = createBaseTableWithProps(conn);
+            createdTablesAndViews.put(baseTableName, new HashSet<String>());
+            Set<String> indexes = createdTablesAndViews.get(baseTableName);
+            indexes.add(createIndexTable(conn, baseTableName, PTable.IndexType.GLOBAL).getSecond());
+            indexes.add(createIndexTable(conn, baseTableName, PTable.IndexType.LOCAL).getFirst());
+
+            viewName = createViewOnBaseTableOrView(conn, baseTableName);
+            createdTablesAndViews.put(viewName, new HashSet<String>());
+            indexes = createdTablesAndViews.get(viewName);
+            indexes.add(createIndexTable(conn, viewName, PTable.IndexType.GLOBAL).getSecond());
+
+            viewName2 = createViewOnBaseTableOrView(conn, viewName);
+            createdTablesAndViews.put(viewName2, new HashSet<String>());
+            indexes = createdTablesAndViews.get(viewName2);
+            indexes.add(createIndexTable(conn, viewName2, PTable.IndexType.LOCAL).getFirst());
+
+            // Intentionally make UPDATE_CACHE_FREQUENCY out of sync for indexes
+            PreparedStatement stmt = conn.prepareStatement(UPSERT_UPDATE_CACHE_FREQUENCY);
+            for (String tableOrViewName : createdTablesAndViews.keySet()) {
+                base = PhoenixRuntime.getTable(conn, tableOrViewName);
+                for (String indexTableName : createdTablesAndViews.get(tableOrViewName)) {
+                    index = PhoenixRuntime.getTable(conn, indexTableName);
+                    PName tenantId = index.getTenantId();
+                    stmt.setString(1, tenantId == null ? null : tenantId.getString());
+                    stmt.setString(2, index.getSchemaName().getString());
+                    stmt.setString(3, index.getTableName().getString());
+                    stmt.setLong(4, base.getType() == PTableType.TABLE ?
+                            MODIFIED_UPDATE_CACHE_FREQUENCY : MODIFIED_UPDATE_CACHE_FREQUENCY_VIEWS);
+                    stmt.addBatch();
+                }
+            }
+            stmt.executeBatch();
+            conn.commit();
+
+            // Clear the server-side cache so that we get the latest built PTables
+            conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache();
+            // Verify that the modified values are reflected
+            for (String tableOrViewName : createdTablesAndViews.keySet()) {
+                assertUpdateCacheFreq(conn, tableOrViewName, baseTableName.equals(tableOrViewName) ?
+                        INITIAL_UPDATE_CACHE_FREQUENCY : INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS);
+                for (String indexName : createdTablesAndViews.get(tableOrViewName)) {
+                    assertUpdateCacheFreq(conn, indexName, baseTableName.equals(tableOrViewName) ?
+                            MODIFIED_UPDATE_CACHE_FREQUENCY : MODIFIED_UPDATE_CACHE_FREQUENCY_VIEWS);
+                }
+            }
+
+            PhoenixConnection upgradeConn = conn.unwrap(PhoenixConnection.class);
+            upgradeConn.setRunningUpgrade(true);
+            syncUpdateCacheFreqAllIndexes(upgradeConn,
+                    PhoenixRuntime.getTableNoCache(conn, baseTableName));
+
+            conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache();
+            // Verify that indexes have the synced values for UPDATE_CACHE_FREQUENCY
+            for (String tableOrViewName : createdTablesAndViews.keySet()) {
+                long expectedVal = baseTableName.equals(tableOrViewName) ?
+                        INITIAL_UPDATE_CACHE_FREQUENCY : INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS;
+                assertUpdateCacheFreq(conn, tableOrViewName, expectedVal);
+                for (String indexOnTableOrView : createdTablesAndViews.get(tableOrViewName)) {
+                    assertUpdateCacheFreq(conn, indexOnTableOrView, expectedVal);
+                }
+            }
+        }
+    }
+
     /**
      * Helper method to modify the synced properties for a column family descriptor
      * @param cfd The column family descriptor object
@@ -417,9 +500,9 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
         conn.createStatement().execute("create table " + tableName
                 + " (id INTEGER not null primary key, type varchar(5), "
                 + COL_FAM1 + ".name varchar(10), " + COL_FAM2 + ".flag boolean) "
-                + "TTL=" + INITIAL_TTL_VALUE + ",KEEP_DELETED_CELLS="
-                + INITIAL_KEEP_DELETED_CELLS_VALUE
-                + ",REPLICATION_SCOPE=" + INITIAL_REPLICATION_SCOPE_VALUE);
+                + "TTL=" + INITIAL_TTL_VALUE + ",KEEP_DELETED_CELLS=" + INITIAL_KEEP_DELETED_CELLS_VALUE
+                + ",REPLICATION_SCOPE=" + INITIAL_REPLICATION_SCOPE_VALUE
+                + ",UPDATE_CACHE_FREQUENCY=" + INITIAL_UPDATE_CACHE_FREQUENCY);
         return tableName;
     }
 
@@ -429,34 +512,43 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
      * @param baseTableName Name of the HBase base table on which to create an index
      * @param indexType LOCAL, GLOBAL or if we pass in null as the indexType,
      *                 we create a view and an index on that view for the given base table
-     * @return The physical HBase table corresponding to the index created
+     * @return A pair consisting of the index name and the name of the physical HBase table
+     * corresponding to the index created
      * @throws SQLException
      */
-    private String createIndexTable(Connection conn, String baseTableName,
+    private Pair<String,String> createIndexTable(Connection conn, String baseTableName,
             PTable.IndexType indexType) throws SQLException {
         // Create a view on top of the base table and then an index on that view
         if (indexType == null) {
-            String viewName = "VIEW_" + baseTableName + "_" + generateUniqueName();
+            String viewName = createViewOnBaseTableOrView(conn, baseTableName);
             String viewIndexName = VIEW_INDEX_TABLE_PREFIX + baseTableName;
-            conn.createStatement().execute("create view " + viewName
-                    + " (new_col SMALLINT) as select * from " + baseTableName + " where id > 1");
             conn.createStatement().execute("create index view_index_" + generateUniqueName()
                     + " on " + viewName + " (flag)");
-            return viewIndexName;
+            return new Pair<>(viewIndexName, viewIndexName);
         }
         switch(indexType) {
         case LOCAL:
             String localIndexName = baseTableName + "_LOCAL_" + generateUniqueName();
             conn.createStatement().execute(
                     "create local index " + localIndexName + " on " + baseTableName + "(flag)");
-            return baseTableName;
+            return new Pair<>(localIndexName, baseTableName);
         case GLOBAL:
             String globalIndexName = baseTableName + "_GLOBAL_" + generateUniqueName();
             conn.createStatement()
                     .execute("create index " + globalIndexName + " on " + baseTableName + "(name)");
-            return globalIndexName;
+            return new Pair<>(globalIndexName, globalIndexName);
+        default:
+            return new Pair<>(baseTableName, baseTableName);
         }
-        return baseTableName;
+    }
+
+    private String createViewOnBaseTableOrView(Connection conn, String baseTableOrView) throws SQLException {
+        String viewName = "VIEW_" + baseTableOrView + "_" + generateUniqueName();
+        conn.createStatement().execute("create view " + viewName
+                + " (" + generateUniqueName() + " SMALLINT) as select * from "
+                + baseTableOrView + " where id > 1 UPDATE_CACHE_FREQUENCY="
+                + INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS);
+        return viewName;
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
index 21fa312..f797b70 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
@@ -19,6 +19,8 @@ package org.apache.phoenix.end2end.index;
 
 import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA;
 import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.apache.phoenix.exception.SQLExceptionCode.CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX;
+import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_UPDATE_CACHE_FREQUENCY;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -675,15 +677,12 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT {
         }
     }
 
-
-
     @Test
     public void testIndexAlterPhoenixProperty() throws Exception {
         Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
         Connection conn = DriverManager.getConnection(getUrl(), props);
         String testTable = generateUniqueName();
 
-
         String ddl = "create table " + testTable  + " (k varchar primary key, v1 varchar)";
         Statement stmt = conn.createStatement();
         stmt.execute(ddl);
@@ -703,6 +702,123 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT {
         assertEquals(20,rs.getInt(1));
     }
 
+    @Test
+    public void testCreateIndexSetUpdateCacheFreqFails() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String testTable = generateUniqueName();
+
+        String ddl = "CREATE TABLE " + testTable  + " (k varchar primary key, v1 varchar)";
+        Statement stmt = conn.createStatement();
+        stmt.execute(ddl);
+        String indexName = "IDX_" + generateUniqueName();
+
+        ddl = "CREATE INDEX " + indexName + " ON " + testTable  + " (v1) " +
+                "UPDATE_CACHE_FREQUENCY=10000";
+        try {
+            stmt.execute(ddl);
+            fail("Should fail trying to set UPDATE_CACHE_FREQUENCY when creating an index");
+        } catch (SQLException sqlE) {
+            assertEquals("Unexpected error occurred",
+                    CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX.getErrorCode(), sqlE.getErrorCode());
+        }
+    }
+
+    @Test
+    public void testIndexGetsUpdateCacheFreqFromBaseTable() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String testTable = generateUniqueName();
+
+        long updateCacheFreq = 10000;
+        String ddl = "CREATE TABLE " + testTable  + " (k varchar primary key, v1 varchar) " +
+                "UPDATE_CACHE_FREQUENCY=" + updateCacheFreq;
+        Statement stmt = conn.createStatement();
+        stmt.execute(ddl);
+
+        String localIndex = "LOCAL_" + generateUniqueName();
+        String globalIndex = "GLOBAL_" + generateUniqueName();
+
+        ddl = "CREATE LOCAL INDEX " + localIndex + " ON " + testTable  + " (v1) ";
+        stmt.execute(ddl);
+        ddl = "CREATE INDEX " + globalIndex + " ON " + testTable  + " (v1) ";
+        stmt.execute(ddl);
+
+        // Check that local and global index both have the propagated UPDATE_CACHE_FREQUENCY value
+        assertUpdateCacheFreq(conn, testTable, updateCacheFreq);
+        assertUpdateCacheFreq(conn, localIndex, updateCacheFreq);
+        assertUpdateCacheFreq(conn, globalIndex, updateCacheFreq);
+    }
+
+    @Test
+    public void testAlterTablePropagatesUpdateCacheFreqToIndexes() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String testTable = generateUniqueName();
+
+        String ddl = "CREATE TABLE " + testTable  + " (k varchar primary key, v1 varchar) ";
+        Statement stmt = conn.createStatement();
+        stmt.execute(ddl);
+
+        String localIndex = "LOCAL_" + generateUniqueName();
+        String globalIndex = "GLOBAL_" + generateUniqueName();
+
+        ddl = "CREATE LOCAL INDEX " + localIndex + " ON " + testTable  + " (v1) ";
+        stmt.execute(ddl);
+        ddl = "CREATE INDEX " + globalIndex + " ON " + testTable  + " (v1) ";
+        stmt.execute(ddl);
+
+        assertUpdateCacheFreq(conn, testTable, DEFAULT_UPDATE_CACHE_FREQUENCY);
+        assertUpdateCacheFreq(conn, localIndex, DEFAULT_UPDATE_CACHE_FREQUENCY);
+        assertUpdateCacheFreq(conn, globalIndex, DEFAULT_UPDATE_CACHE_FREQUENCY);
+
+        // Alter UPDATE_CACHE_FREQUENCY on the base table
+        long updateCacheFreq = 10000;
+        ddl = "ALTER TABLE " + testTable + " SET UPDATE_CACHE_FREQUENCY=" + updateCacheFreq;
+        stmt.execute(ddl);
+
+        // Check that local and global index both have the propagated UPDATE_CACHE_FREQUENCY value
+        assertUpdateCacheFreq(conn, testTable, updateCacheFreq);
+        assertUpdateCacheFreq(conn, localIndex, updateCacheFreq);
+        assertUpdateCacheFreq(conn, globalIndex, updateCacheFreq);
+    }
+
+    @Test
+    public void testIndexAlterUpdateCacheFreqFails() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        String testTable = generateUniqueName();
+
+        String ddl = "CREATE TABLE " + testTable  + " (k varchar primary key, v1 varchar)";
+        Statement stmt = conn.createStatement();
+        stmt.execute(ddl);
+
+        String localIndex = "LOCAL_" + generateUniqueName();
+        String globalIndex = "GLOBAL_" + generateUniqueName();
+
+        ddl = "CREATE LOCAL INDEX " + localIndex + " ON " + testTable  + " (v1) ";
+        stmt.execute(ddl);
+        ddl = "CREATE INDEX " + globalIndex + " ON " + testTable  + " (v1) ";
+        stmt.execute(ddl);
+
+        try {
+            stmt.execute("ALTER INDEX " + localIndex + " ON " + testTable +
+                    " ACTIVE SET UPDATE_CACHE_FREQUENCY=NEVER");
+            fail("Should fail trying to alter UPDATE_CACHE_FREQUENCY on index");
+        } catch (SQLException sqlE) {
+            assertEquals("Unexpected error occurred",
+                    CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX.getErrorCode(), sqlE.getErrorCode());
+        }
+
+        try {
+            stmt.execute("ALTER INDEX " + globalIndex + " ON " + testTable +
+                    " ACTIVE SET UPDATE_CACHE_FREQUENCY=NEVER");
+            fail("Should fail trying to alter UPDATE_CACHE_FREQUENCY on index");
+        } catch (SQLException sqlE) {
+            assertEquals("Unexpected error occurred",
+                    CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX.getErrorCode(), sqlE.getErrorCode());
+        }
+    }
 
     @Test
     public void testIndexAlterHBaseProperty() throws Exception {
@@ -718,10 +834,10 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT {
         ddl = "CREATE INDEX " + indexName + " ON " + testTable  + " (v1) ";
         stmt.execute(ddl);
 
-        conn.createStatement().execute("ALTER INDEX "+indexName+" ON " + testTable +" ACTIVE SET DISABLE_WAL=false");
-        asssertIsWALDisabled(conn,indexName,false);
         conn.createStatement().execute("ALTER INDEX "+indexName+" ON " + testTable +" ACTIVE SET DISABLE_WAL=true");
         asssertIsWALDisabled(conn,indexName,true);
+        conn.createStatement().execute("ALTER INDEX "+indexName+" ON " + testTable +" ACTIVE SET DISABLE_WAL=false");
+        asssertIsWALDisabled(conn,indexName,false);
     }
 
     private static void asssertIsWALDisabled(Connection conn, String fullTableName, boolean expectedValue) throws SQLException {
@@ -729,4 +845,23 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT {
         assertEquals(expectedValue, pconn.getTable(new PTableKey(pconn.getTenantId(), fullTableName)).isWALDisabled());
     }
 
+    /**
+     * Helper method to assert the value of UPDATE_CACHE_FREQUENCY for a table/index/view
+     * @param conn Phoenix connection
+     * @param name table/view/index name
+     * @param expectedUpdateCacheFreq expected value of UPDATE_CACHE_FREQUENCY
+     * @throws SQLException
+     */
+    public static void assertUpdateCacheFreq(Connection conn, String name,
+            long expectedUpdateCacheFreq) throws SQLException {
+        ResultSet rs = conn.createStatement().executeQuery(
+                "select UPDATE_CACHE_FREQUENCY from SYSTEM.\"CATALOG\" where TABLE_NAME='" +
+                        name + "'");
+        assertTrue(rs.next());
+        assertEquals("Mismatch found for " + name, expectedUpdateCacheFreq, rs.getLong(1));
+        assertEquals("Mismatch in UPDATE_CACHE_FREQUENCY for PTable of " + name,
+                expectedUpdateCacheFreq, PhoenixRuntime.getTableNoCache(conn, name)
+                        .getUpdateCacheFrequency());
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
index 0ddbed3..2959b99 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.rpc;
 
 import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA;
 import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
 import static org.mockito.Matchers.anyLong;
 import static org.mockito.Matchers.eq;
@@ -73,7 +74,7 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
                 String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + tableName;
                 Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES));
                 conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + "TRANSACTIONAL=true,TRANSACTION_PROVIDER='" + provider + "'");
-                helpTestUpdateCache(fullTableName, new int[] {1, 1});
+                helpTestUpdateCache(fullTableName, new int[] {1, 1}, false);
             }
         }
     }
@@ -84,14 +85,14 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
         String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + tableName;
         Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES));
         conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA);
-        helpTestUpdateCache(fullTableName, new int[] {1, 3});
+        helpTestUpdateCache(fullTableName, new int[] {1, 3}, false);
     }
 	
     @Test
     public void testUpdateCacheForNonTxnSystemTable() throws Exception {
         String fullTableName = "\""+ QueryConstants.SYSTEM_SCHEMA_NAME + "\""+ QueryConstants.NAME_SEPARATOR + generateUniqueName();
         setupSystemTable(fullTableName);
-        helpTestUpdateCache(fullTableName, new int[] {0, 0});
+        helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
     }
     
     @Test
@@ -104,7 +105,7 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
             conn.createStatement().execute(
             "alter table " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=NEVER");
         }
-        helpTestUpdateCache(fullTableName, new int[] {0, 0});
+        helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
     }
     
     @Test
@@ -114,7 +115,7 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
         try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
             conn.createStatement().execute("CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=always");
         }
-        helpTestUpdateCache(fullTableName, new int[] {1, 3});
+        helpTestUpdateCache(fullTableName, new int[] {1, 3}, false);
     }
     
     @Test
@@ -124,9 +125,9 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
         try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
             conn.createStatement().execute("CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=" + 10000);
         }
-        helpTestUpdateCache(fullTableName, new int[] {0, 0});
+        helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
         Thread.sleep(10000);
-        helpTestUpdateCache(fullTableName, new int[] {1, 0});
+        helpTestUpdateCache(fullTableName, new int[] {1, 0}, false);
     }
     
     @Test
@@ -136,14 +137,55 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
         try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
             conn.createStatement().execute("CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=never");
         }
-        helpTestUpdateCache(fullTableName, new int[] {0, 0});
+        helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
         try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
             conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=ALWAYS");
         }
-        helpTestUpdateCache(fullTableName, new int[] {1, 3});
+        helpTestUpdateCache(fullTableName, new int[] {1, 3}, false);
+    }
+
+    @Test
+    public void testUpdateCacheFreqPropagatedToIndexes() throws Exception {
+        String baseTableName = generateUniqueName();
+        String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + baseTableName;
+        String localIndex = "LOCAL_" + baseTableName;
+        String globalIndex = "GLOBAL_" + baseTableName;
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+            conn.createStatement().execute("CREATE TABLE " + fullTableName +
+                    TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=never");
+
+            // Create local and global indexes on the base table
+            conn.createStatement().execute("CREATE LOCAL INDEX " + localIndex
+                    + " on " + fullTableName + " (a.date1, b.varchar_col2)");
+            conn.createStatement().execute("CREATE INDEX " + globalIndex + " on "
+                    + fullTableName + " (a.int_col1, a.long_col1)");
+        }
+
+        // The indexes should have got the UPDATE_CACHE_FREQUENCY value of their base table
+        helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
+        helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + localIndex,
+                new int[] {0}, true);
+        helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + globalIndex,
+                new int[] {0}, true);
+
+        // Now alter the UPDATE_CACHE_FREQUENCY value of the base table
+        try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+            conn.createStatement()
+                    .execute("ALTER TABLE " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=ALWAYS");
+        }
+        // Even the indexes should now have the modified value of UPDATE_CACHE_FREQUENCY
+        // Note that when we query the base table, during query plan generation, we make 2 getTable
+        // requests (to retrieve the base table) for each index of the base table
+        helpTestUpdateCache(fullTableName, new int[] {1, 15}, false);
+        helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + localIndex,
+                new int[] {3}, true);
+        helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + globalIndex,
+                new int[] {3}, true);
     }
     
-	private static void helpTestUpdateCache(String fullTableName, int[] expectedRPCs) throws Exception {
+	private static void helpTestUpdateCache(String fullTableName, int[] expectedRPCs,
+            boolean skipUpsertForIndexes) throws Exception {
 	    String tableName = SchemaUtil.getTableNameFromFullName(fullTableName);
 	    String schemaName = SchemaUtil.getSchemaNameFromFullName(fullTableName);
 		String selectSql = "SELECT * FROM "+fullTableName;
@@ -154,46 +196,31 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
 		Connection conn = connectionQueryServices.connect(getUrl(), props);
 		try {
 			conn.setAutoCommit(false);
-	        String upsert = "UPSERT INTO " + fullTableName + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
-	        PreparedStatement stmt = conn.prepareStatement(upsert);
-			// upsert three rows
-	        TestUtil.setRowKeyColumns(stmt, 1);
-			stmt.execute();
-			TestUtil.setRowKeyColumns(stmt, 2);
-			stmt.execute();
-			TestUtil.setRowKeyColumns(stmt, 3);
-			stmt.execute();
-			conn.commit();
-            int numUpsertRpcs = expectedRPCs[0];
-			// verify only 0 or 1 rpc to fetch table metadata, 
-            verify(connectionQueryServices, times(numUpsertRpcs)).getTable((PName) isNull(),
-                eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)),
-                anyLong(), anyLong(), eq(false), eq(false), (PTable)isNull());
-            reset(connectionQueryServices);
-            
-            ResultSet rs = conn.createStatement().executeQuery(selectSql);
-			TestUtil.validateRowKeyColumns(rs, 1);
-			TestUtil.validateRowKeyColumns(rs, 2);
-			TestUtil.validateRowKeyColumns(rs, 3);
-	        assertFalse(rs.next());
-	        
-	        rs = conn.createStatement().executeQuery(selectSql);
-	        TestUtil.validateRowKeyColumns(rs, 1);
-	        TestUtil.validateRowKeyColumns(rs, 2);
-	        TestUtil.validateRowKeyColumns(rs, 3);
-	        assertFalse(rs.next());
-	        
-	        rs = conn.createStatement().executeQuery(selectSql);
-	        TestUtil.validateRowKeyColumns(rs, 1);
-	        TestUtil.validateRowKeyColumns(rs, 2);
-	        TestUtil.validateRowKeyColumns(rs, 3);
-	        assertFalse(rs.next());
-	        
+            if (!skipUpsertForIndexes) {
+                String upsert = "UPSERT INTO " + fullTableName + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
+                PreparedStatement stmt = conn.prepareStatement(upsert);
+                // upsert three rows
+                for (int i=0; i<3; i++) {
+                    TestUtil.setRowKeyColumns(stmt, i);
+                    stmt.execute();
+                }
+                conn.commit();
+                int numUpsertRpcs = expectedRPCs[0];
+                // verify only 0 or 1 rpc to fetch table metadata,
+                verify(connectionQueryServices, times(numUpsertRpcs)).getTable((PName) isNull(),
+                        eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)),
+                        anyLong(), anyLong(), eq(false), eq(false), (PTable)isNull());
+                reset(connectionQueryServices);
+            }
+            validateSelectRowKeyCols(conn, selectSql, skipUpsertForIndexes);
+            validateSelectRowKeyCols(conn, selectSql, skipUpsertForIndexes);
+            validateSelectRowKeyCols(conn, selectSql, skipUpsertForIndexes);
+
 	        // for non-transactional tables without a scn : verify one rpc to getTable occurs *per* query
             // for non-transactional tables with a scn : verify *only* one rpc occurs
             // for transactional tables : verify *only* one rpc occurs
 	        // for non-transactional, system tables : verify no rpc occurs
-            int numRpcs = expectedRPCs[1]; 
+            int numRpcs = skipUpsertForIndexes ? expectedRPCs[0] : expectedRPCs[1];
             verify(connectionQueryServices, times(numRpcs)).getTable((PName) isNull(),
                 eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)),
                 anyLong(), anyLong(), eq(false), eq(false), (PTable)isNull());
@@ -202,4 +229,19 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
         	conn.close();
         }
 	}
+
+	private static void validateSelectRowKeyCols(Connection conn, String selectSql,
+            boolean skipUpsertForIndexes) throws SQLException {
+        ResultSet rs = conn.createStatement().executeQuery(selectSql);
+        if (skipUpsertForIndexes) {
+            for (int i=0; i<3; i++) {
+                assertTrue(rs.next());
+            }
+        } else {
+            for (int i=0; i<3; i++) {
+                TestUtil.validateRowKeyColumns(rs, i);
+            }
+        }
+        assertFalse(rs.next());
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index 6696521..5bffed5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -23,6 +23,7 @@ import java.util.Map;
 
 import org.apache.phoenix.coprocessor.MetaDataProtocol;
 import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.schema.AmbiguousColumnException;
@@ -263,7 +264,7 @@ public enum SQLExceptionCode {
     NO_LOCAL_INDEX_ON_TABLE_WITH_IMMUTABLE_ROWS(1048,"43A05","Local indexes aren't allowed on tables with immutable rows."),
     COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY(1049, "43A06", "Column family not allowed for table properties."),
     COLUMN_FAMILY_NOT_ALLOWED_FOR_PROPERTY(1050, "43A07", "Setting  or altering any of the following properties: "
-            + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES.toString()
+            + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES.toString()
             + " for a column family is not supported since they must be kept in sync. You can only set these properties for the entire table."),
     CANNOT_ALTER_PROPERTY(1051, "43A08", "Property can be specified or changed only when creating a table."),
     CANNOT_SET_PROPERTY_FOR_COLUMN_NOT_ADDED(1052, "43A09", "Property cannot be specified for a column family that is not being added or modified."),
@@ -308,7 +309,9 @@ public enum SQLExceptionCode {
     TTL_UNSUPPORTED_FOR_TXN_TABLE(10947, "44A28", "TTL is not supported for"),
     CANNOT_CREATE_LOCAL_INDEX_FOR_TXN_TABLE(10948, "44A29", "Local indexes cannot be created for"),
     CANNOT_SET_OR_ALTER_PROPERTY_FOR_INDEX(10949, "44A30", "Cannot set or alter the following properties on an index: "
-            + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES.toString()),
+            + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES.toString()),
+    CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX(10950, "44A31", "Cannot set or alter "
+            + PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " on an index"),
 
     /** Sequence related */
     SEQUENCE_ALREADY_EXIST(1200, "42Z00", "Sequence already exists.", new Factory() {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index ebcb7b9..1381247 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -2519,7 +2519,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
      * Keep the TTL, KEEP_DELETED_CELLS and REPLICATION_SCOPE properties of new column families
      * in sync with the existing column families. Note that we use the new values for these properties in case they
      * are passed from our alter table command, if not, we use the default column family's value for each property
-     * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES}
+     * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES}
      * @param allFamiliesProps Map of all column family properties
      * @param table original table
      * @param tableDesc new table descriptor
@@ -2565,7 +2565,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     /**
      * Set the new values for properties that are to be kept in sync amongst those column families of the table which are
      * not referenced in the context of our alter table command, including the local index column family if it exists
-     * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES}
+     * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES}
      * @param tableDesc original table descriptor
      * @param allFamiliesProps Map of all column family properties
      * @param newTTL new value of TTL
@@ -2587,7 +2587,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     /**
      * Set properties to be kept in sync for global indexes of a table, as well as
      * the physical table corresponding to indexes created on views of a table
-     * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES} and
+     * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES} and
      * @param table base table
      * @param tableAndIndexDescriptorMappings old to new table descriptor mappings
      * @param applyPropsToAllIndexesDefaultCF new properties to apply to all index column families

http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 6563c40..df0e5d4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -272,6 +272,7 @@ public class MetaDataClient {
                     TABLE_NAME + "," +
                     SYNC_INDEX_CREATED_DATE + " " + PDate.INSTANCE.getSqlTypeName() +
                     ") VALUES (?, ?, ?, ?)";
+
     private static final String CREATE_TABLE =
             "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " +
                     TENANT_ID + "," +
@@ -1147,7 +1148,7 @@ public class MetaDataClient {
     /**
      * Populate properties for the table and common properties for all column families of the table
      * @param statementProps Properties specified in SQL statement
-     * @param tableProps Properties for an HTableDescriptor
+     * @param tableProps Properties for an HTableDescriptor and Phoenix Table Properties
      * @param commonFamilyProps Properties common to all column families
      * @param tableType Used to distinguish between index creation vs. base table creation paths
      * @throws SQLException
@@ -1164,9 +1165,17 @@ public class MetaDataClient {
                             .setMessage("Property: " + prop.getFirst()).build()
                             .buildException();
                 }
+                // HTableDescriptor property or Phoenix Table Property
                 if (defaultDescriptor.getValue(Bytes.toBytes(prop.getFirst())) == null) {
+                    // See PHOENIX-4891
+                    if (tableType == PTableType.INDEX && UPDATE_CACHE_FREQUENCY.equals(prop.getFirst())) {
+                        throw new SQLExceptionInfo.Builder(
+                                SQLExceptionCode.CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX)
+                                .build()
+                                .buildException();
+                    }
                     tableProps.put(prop.getFirst(), prop.getSecond());
-                } else {
+                } else { // HColumnDescriptor property
                     commonFamilyProps.put(prop.getFirst(), prop.getSecond());
                 }
             }
@@ -2127,8 +2136,11 @@ public class MetaDataClient {
             }
             long updateCacheFrequency = connection.getQueryServices().getProps().getLong(
                 QueryServices.DEFAULT_UPDATE_CACHE_FREQUENCY_ATRRIB, QueryServicesOptions.DEFAULT_UPDATE_CACHE_FREQUENCY);
+            if (tableType == PTableType.INDEX && parent != null) {
+                updateCacheFrequency = parent.getUpdateCacheFrequency();
+            }
             Long updateCacheFrequencyProp = (Long) TableProperty.UPDATE_CACHE_FREQUENCY.getValue(tableProps);
-            if (updateCacheFrequencyProp != null) {
+            if (tableType != PTableType.INDEX && updateCacheFrequencyProp != null) {
                 updateCacheFrequency = updateCacheFrequencyProp;
             }
             String autoPartitionSeq = (String) TableProperty.AUTO_PARTITION_SEQ.getValue(tableProps);
@@ -3613,9 +3625,13 @@ public class MetaDataClient {
                     }
                 }
 
-                if (!table.getIndexes().isEmpty() && (numPkColumnsAdded>0 || metaProperties.getNonTxToTx())) {
+                if (!table.getIndexes().isEmpty() &&
+                        (numPkColumnsAdded>0 || metaProperties.getNonTxToTx() ||
+                                metaPropertiesEvaluated.getUpdateCacheFrequency() != null)) {
                     for (PTable index : table.getIndexes()) {
-                        incrementTableSeqNum(index, index.getType(), numPkColumnsAdded, metaProperties.getNonTxToTx() ? Boolean.TRUE : null, metaPropertiesEvaluated.getUpdateCacheFrequency());
+                        incrementTableSeqNum(index, index.getType(), numPkColumnsAdded,
+                                metaProperties.getNonTxToTx() ? Boolean.TRUE : null,
+                                metaPropertiesEvaluated.getUpdateCacheFrequency());
                     }
                     tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
                     connection.rollback();
@@ -4573,6 +4589,13 @@ public class MetaDataClient {
         }
 
         if (metaProperties.getUpdateCacheFrequencyProp() != null) {
+            // See PHOENIX-4891
+            if (table.getType() == PTableType.INDEX) {
+                throw new SQLExceptionInfo.Builder(
+                        SQLExceptionCode.CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX)
+                        .build()
+                        .buildException();
+            }
             if (metaProperties.getUpdateCacheFrequencyProp().longValue() != table.getUpdateCacheFrequency()) {
                 metaPropertiesEvaluated.setUpdateCacheFrequency(metaProperties.getUpdateCacheFrequencyProp());
                 changingPhoenixTableProperty = true;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index df7530c..8cf514f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -109,7 +109,7 @@ public class MetaDataUtil {
     public static final byte[] DATA_TABLE_NAME_PROP_BYTES = Bytes.toBytes(DATA_TABLE_NAME_PROP_NAME);
 
     // See PHOENIX-3955
-    public static final List<String> SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES = ImmutableList.of(
+    public static final List<String> SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES = ImmutableList.of(
             HColumnDescriptor.TTL,
             HColumnDescriptor.KEEP_DELETED_CELLS,
             HColumnDescriptor.REPLICATION_SCOPE);
@@ -725,13 +725,13 @@ public class MetaDataUtil {
     }
 
     public static boolean propertyNotAllowedToBeOutOfSync(String colFamProp) {
-        return SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES.contains(colFamProp);
+        return SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES.contains(colFamProp);
     }
 
     public static Map<String, Object> getSyncedProps(HColumnDescriptor defaultCFDesc) {
         Map<String, Object> syncedProps = new HashMap<>();
         if (defaultCFDesc != null) {
-            for (String propToKeepInSync: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+            for (String propToKeepInSync: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
                 syncedProps.put(propToKeepInSync, Bytes.toString(
                         defaultCFDesc.getValue(Bytes.toBytes(propToKeepInSync))));
             }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
index ede14b8..8d0aca1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
@@ -46,6 +46,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
 import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
 import static org.apache.phoenix.query.QueryConstants.DIVERGED_VIEW_BASE_COLUMN_COUNT;
@@ -140,6 +141,14 @@ public class UpgradeUtil {
             + "(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, BASE_COLUMN_COUNT) "
             + "VALUES (?, ?, ?, ?, ?, ?) ";
 
+    public static final String UPSERT_UPDATE_CACHE_FREQUENCY =
+            "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " +
+            TENANT_ID + "," +
+            TABLE_SCHEM + "," +
+            TABLE_NAME + "," +
+            UPDATE_CACHE_FREQUENCY +
+            ") VALUES (?, ?, ?, ?)";
+
     public static String SELECT_BASE_COLUMN_COUNT_FROM_HEADER_ROW = "SELECT "
             + "BASE_COLUMN_COUNT "
             + "FROM \"SYSTEM\".CATALOG "
@@ -1227,7 +1236,7 @@ public class UpgradeUtil {
         for (HColumnDescriptor currentColFam: tableDesc.getColumnFamilies()) {
             if (!currentColFam.equals(defaultColFam)) {
                 HColumnDescriptor newColFamDesc = new HColumnDescriptor(currentColFam);
-                for (String prop: MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+                for (String prop: MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
                     String existingPropVal = Bytes.toString(currentColFam.getValue(Bytes.toBytes(prop)));
                     String expectedPropVal = syncedProps.get(prop).toString();
                     if (existingPropVal == null || !existingPropVal.toLowerCase().equals(expectedPropVal.toLowerCase())) {
@@ -1299,6 +1308,71 @@ public class UpgradeUtil {
         }
     }
 
+    private static void syncUpdateCacheFreqForIndexesOfTable(PTable baseTable,
+            PreparedStatement stmt) throws SQLException {
+        for (PTable index : baseTable.getIndexes()) {
+            if (index.getUpdateCacheFrequency() == baseTable.getUpdateCacheFrequency()) {
+                continue;
+            }
+            stmt.setString(2, index.getSchemaName().getString());
+            stmt.setString(3, index.getTableName().getString());
+            stmt.setLong(4, baseTable.getUpdateCacheFrequency());
+            stmt.addBatch();
+        }
+    }
+
+    /**
+     * See PHOENIX-4891. We set the UPDATE_CACHE_FREQUENCY of indexes to be same as their parent.
+     * We do this for both physical base tables as well as views
+     * @param conn Phoenix Connection object
+     * @param table PTable corresponding to a physical base table
+     * @throws SQLException
+     * @throws IOException
+     */
+    public static void syncUpdateCacheFreqAllIndexes(PhoenixConnection conn, PTable table)
+    throws SQLException, IOException {
+        // Use own connection with max time stamp to be able to read all data from SYSTEM.CATALOG
+        try(PhoenixConnection newConn = new PhoenixConnection(conn, HConstants.LATEST_TIMESTAMP)) {
+            // Clear the server-side cache so that we get the latest built PTables
+            newConn.unwrap(PhoenixConnection.class).getQueryServices().clearCache();
+            byte[] tenantId = newConn.getTenantId() != null ?
+                    newConn.getTenantId().getBytes() : null;
+
+            PreparedStatement stmt =
+                    newConn.prepareStatement(UPSERT_UPDATE_CACHE_FREQUENCY);
+            stmt.setString(1, Bytes.toString(tenantId));
+            syncUpdateCacheFreqForIndexesOfTable(table, stmt);
+
+            TableViewFinderResult childViewsResult = new TableViewFinderResult();
+            try (Table childLinkTable = newConn.getQueryServices()
+                    .getTable(SchemaUtil.getPhysicalName(
+                            PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES,
+                            newConn.getQueryServices().getProps())
+                            .getName())) {
+                ViewFinder.findAllRelatives(childLinkTable, tenantId,
+                        table.getSchemaName().getBytes(), table.getTableName().getBytes(),
+                        LinkType.CHILD_TABLE, childViewsResult);
+
+                // Iterate over the chain of child views
+                for (TableInfo tableInfo: childViewsResult.getLinks()) {
+                    PTable view;
+                    String viewName = SchemaUtil.getTableName(tableInfo.getSchemaName(),
+                            tableInfo.getTableName());
+                    try {
+                        view = PhoenixRuntime.getTable(newConn, viewName);
+                    } catch (TableNotFoundException e) {
+                        // Ignore
+                        logger.warn("Error getting PTable for view: " + viewName);
+                        continue;
+                    }
+                    syncUpdateCacheFreqForIndexesOfTable(view, stmt);
+                }
+            }
+            stmt.executeBatch();
+            newConn.commit();
+        }
+    }
+
     /**
      * Make sure that all tables have necessary column family properties in sync
      * with each other and also in sync with all the table's indexes
@@ -1316,7 +1390,7 @@ public class UpgradeUtil {
                 // Ignore physical view index tables since we handle them for each base table already
                 continue;
             }
-            PTable table = null;
+            PTable table;
             String tableName = origTableDesc.getTableName().getNameAsString();
             try {
                 table = PhoenixRuntime.getTable(conn, tableName);
@@ -1329,6 +1403,7 @@ public class UpgradeUtil {
                 // Ignore global index tables since we handle them for each base table already
                 continue;
             }
+            syncUpdateCacheFreqAllIndexes(conn, table);
             HColumnDescriptor defaultColFam = origTableDesc.getFamily(SchemaUtil.getEmptyColumnFamily(table));
             Map<String, Object> syncedProps = MetaDataUtil.getSyncedProps(defaultColFam);
 


[16/22] phoenix git commit: PHOENIX-4996: Refactor PTableImpl to use Builder Pattern

Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index 9f06e04..7939b97 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -36,6 +36,7 @@ import java.util.Map.Entry;
 
 import javax.annotation.Nonnull;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.Delete;
@@ -69,7 +70,6 @@ import org.apache.phoenix.schema.types.PChar;
 import org.apache.phoenix.schema.types.PDataType;
 import org.apache.phoenix.schema.types.PDouble;
 import org.apache.phoenix.schema.types.PFloat;
-import org.apache.phoenix.schema.types.PLong;
 import org.apache.phoenix.schema.types.PVarchar;
 import org.apache.phoenix.transaction.TransactionFactory;
 import org.apache.phoenix.util.ByteUtil;
@@ -102,164 +102,661 @@ import com.google.common.collect.Maps;
 public class PTableImpl implements PTable {
     private static final Integer NO_SALTING = -1;
 
-    private PTableKey key;
-    private PName name;
-    private PName schemaName = PName.EMPTY_NAME;
-    private PName tableName = PName.EMPTY_NAME;
-    private PName tenantId;
-    private PTableType type;
-    private PIndexState state;
-    private long sequenceNumber;
-    private long timeStamp;
-    private long indexDisableTimestamp;
+    private IndexMaintainer indexMaintainer;
+    private ImmutableBytesWritable indexMaintainersPtr;
+
+    private final PTableKey key;
+    private final PName name;
+    private final PName schemaName;
+    private final PName tableName;
+    private final PName tenantId;
+    private final PTableType type;
+    private final PIndexState state;
+    private final long sequenceNumber;
+    private final long timeStamp;
+    private final long indexDisableTimestamp;
     // Have MultiMap for String->PColumn (may need family qualifier)
-    private List<PColumn> pkColumns;
-    private List<PColumn> allColumns;
+    private final List<PColumn> pkColumns;
+    private final List<PColumn> allColumns;
     // columns that were inherited from a parent table but that were dropped in the view
-    private List<PColumn> excludedColumns;
-    private List<PColumnFamily> families;
-    private Map<byte[], PColumnFamily> familyByBytes;
-    private Map<String, PColumnFamily> familyByString;
-    private ListMultimap<String, PColumn> columnsByName;
-    private Map<KVColumnFamilyQualifier, PColumn> kvColumnsByQualifiers;
-    private PName pkName;
-    private Integer bucketNum;
-    private RowKeySchema rowKeySchema;
+    private final List<PColumn> excludedColumns;
+    private final List<PColumnFamily> families;
+    private final Map<byte[], PColumnFamily> familyByBytes;
+    private final Map<String, PColumnFamily> familyByString;
+    private final ListMultimap<String, PColumn> columnsByName;
+    private final Map<KVColumnFamilyQualifier, PColumn> kvColumnsByQualifiers;
+    private final PName pkName;
+    private final Integer bucketNum;
+    private final RowKeySchema rowKeySchema;
     // Indexes associated with this table.
-    private List<PTable> indexes;
+    private final List<PTable> indexes;
     // Data table name that the index is created on.
-    private PName parentName;
-    private PName parentSchemaName;
-    private PName parentTableName;
-    private List<PName> physicalNames;
-    private boolean isImmutableRows;
-    private IndexMaintainer indexMaintainer;
-    private ImmutableBytesWritable indexMaintainersPtr;
-    private PName defaultFamilyName;
-    private String viewStatement;
-    private boolean disableWAL;
-    private boolean multiTenant;
-    private boolean storeNulls;
-    private TransactionFactory.Provider transactionProvider;
-    private ViewType viewType;
-    private PDataType viewIndexType;
-    private Long viewIndexId;
-    private int estimatedSize;
-    private IndexType indexType;
-    private int baseColumnCount;
-    private boolean rowKeyOrderOptimizable; // TODO: remove when required that tables have been upgrade for PHOENIX-2067
-    private boolean hasColumnsRequiringUpgrade; // TODO: remove when required that tables have been upgrade for PHOENIX-2067
-    private int rowTimestampColPos;
-    private long updateCacheFrequency;
-    private boolean isNamespaceMapped;
-    private String autoPartitionSeqName;
-    private boolean isAppendOnlySchema;
-    private ImmutableStorageScheme immutableStorageScheme;
-    private QualifierEncodingScheme qualifierEncodingScheme;
-    private EncodedCQCounter encodedCQCounter;
-    private Boolean useStatsForParallelization;
-
-    public PTableImpl() {
-        this.indexes = Collections.emptyList();
-        this.physicalNames = Collections.emptyList();
-        this.rowKeySchema = RowKeySchema.EMPTY_SCHEMA;
-    }
-    
-    // Constructor used at table creation time
-    public PTableImpl(PName tenantId, String schemaName, String tableName, long timestamp, List<PColumnFamily> families, boolean isNamespaceMapped) {
-        Preconditions.checkArgument(tenantId==null || tenantId.getBytes().length > 0); // tenantId should be null or not empty
-        this.tenantId = tenantId;
-        this.name = PNameFactory.newName(SchemaUtil.getTableName(schemaName, tableName));
-        this.key = new PTableKey(tenantId, name.getString());
-        this.schemaName = PNameFactory.newName(schemaName);
-        this.tableName = PNameFactory.newName(tableName);
-        this.type = PTableType.VIEW;
-        this.viewType = ViewType.MAPPED;
-        this.timeStamp = timestamp;
-        this.pkColumns = this.allColumns = Collections.emptyList();
-        this.rowKeySchema = RowKeySchema.EMPTY_SCHEMA;
-        this.indexes = Collections.emptyList();
-        this.familyByBytes = Maps.newHashMapWithExpectedSize(families.size());
-        this.familyByString = Maps.newHashMapWithExpectedSize(families.size());
-        for (PColumnFamily family : families) {
-            familyByBytes.put(family.getName().getBytes(), family);
-            familyByString.put(family.getName().getString(), family);
-        }
-        this.families = families;
-        this.physicalNames = Collections.emptyList();
-        this.isNamespaceMapped = isNamespaceMapped;
-    }
-    
-    public PTableImpl(PName tenantId, String schemaName, String tableName, long timestamp, List<PColumnFamily> families, boolean isNamespaceMapped, ImmutableStorageScheme storageScheme, QualifierEncodingScheme encodingScheme, Boolean useStatsForParallelization) { // For base table of mapped VIEW
-        Preconditions.checkArgument(tenantId==null || tenantId.getBytes().length > 0); // tenantId should be null or not empty
-        this.tenantId = tenantId;
-        this.name = PNameFactory.newName(SchemaUtil.getTableName(schemaName, tableName));
-        this.key = new PTableKey(tenantId, name.getString());
-        this.schemaName = PNameFactory.newName(schemaName);
-        this.tableName = PNameFactory.newName(tableName);
-        this.type = PTableType.VIEW;
-        this.viewType = ViewType.MAPPED;
-        this.timeStamp = timestamp;
-        this.pkColumns = this.allColumns = Collections.emptyList();
-        this.rowKeySchema = RowKeySchema.EMPTY_SCHEMA;
-        this.indexes = Collections.emptyList();
-        this.familyByBytes = Maps.newHashMapWithExpectedSize(families.size());
-        this.familyByString = Maps.newHashMapWithExpectedSize(families.size());
-        for (PColumnFamily family : families) {
-            familyByBytes.put(family.getName().getBytes(), family);
-            familyByString.put(family.getName().getString(), family);
-        }
-        this.families = families;
-        this.physicalNames = Collections.emptyList();
-        this.isNamespaceMapped = isNamespaceMapped;
-        this.immutableStorageScheme = storageScheme;
-        this.qualifierEncodingScheme = encodingScheme;
-        this.useStatsForParallelization = useStatsForParallelization;
-    }
-    
-    // For indexes stored in shared physical tables
-    public PTableImpl(PName tenantId, PName schemaName, PName tableName, long timestamp, List<PColumnFamily> families, 
-            List<PColumn> columns, List<PName> physicalNames, PDataType viewIndexType, Long viewIndexId, boolean multiTenant, boolean isNamespaceMpped, ImmutableStorageScheme storageScheme, QualifierEncodingScheme qualifierEncodingScheme,
-            EncodedCQCounter encodedCQCounter, Boolean useStatsForParallelization, Integer bucketNum) throws SQLException {
-        this.pkColumns = this.allColumns = Collections.emptyList();
-        this.rowKeySchema = RowKeySchema.EMPTY_SCHEMA;
-        this.indexes = Collections.emptyList();
-        this.familyByBytes = Maps.newHashMapWithExpectedSize(families.size());
-        this.familyByString = Maps.newHashMapWithExpectedSize(families.size());
-        for (PColumnFamily family : families) {
-            familyByBytes.put(family.getName().getBytes(), family);
-            familyByString.put(family.getName().getString(), family);
-        }
-        this.families = families;
-        if (bucketNum!=null) {
-            columns = columns.subList(1, columns.size());
-        }
-        init(tenantId, this.schemaName, this.tableName, PTableType.INDEX, state, timeStamp, sequenceNumber, pkName, bucketNum, columns,
-            this.schemaName, parentTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName,
-            null, disableWAL, multiTenant, storeNulls, viewType, viewIndexType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable,
-            transactionProvider, updateCacheFrequency, indexDisableTimestamp, isNamespaceMpped, null,
-            false, storageScheme, qualifierEncodingScheme, encodedCQCounter, useStatsForParallelization, null);
-    }
-
-    public PTableImpl(long timeStamp) { // For delete marker
-        this(timeStamp, false);
-    }
-
-    public PTableImpl(long timeStamp, boolean isIndex) { // For index delete marker
-        if (isIndex) {
-            this.type = PTableType.INDEX;
-            this.state = PIndexState.INACTIVE;
-        } else {
-            this.type = PTableType.TABLE;
+    private final PName parentName;
+    private final PName parentSchemaName;
+    private final PName parentTableName;
+    private final List<PName> physicalNames;
+    private final boolean isImmutableRows;
+    private final PName defaultFamilyName;
+    private final String viewStatement;
+    private final boolean disableWAL;
+    private final boolean multiTenant;
+    private final boolean storeNulls;
+    private final TransactionFactory.Provider transactionProvider;
+    private final ViewType viewType;
+    private final PDataType viewIndexType;
+    private final Long viewIndexId;
+    private final int estimatedSize;
+    private final IndexType indexType;
+    private final int baseColumnCount;
+    private final boolean rowKeyOrderOptimizable; // TODO: remove when required that tables have been upgrade for PHOENIX-2067
+    private final boolean hasColumnsRequiringUpgrade; // TODO: remove when required that tables have been upgrade for PHOENIX-2067
+    private final int rowTimestampColPos;
+    private final long updateCacheFrequency;
+    private final boolean isNamespaceMapped;
+    private final String autoPartitionSeqName;
+    private final boolean isAppendOnlySchema;
+    private final ImmutableStorageScheme immutableStorageScheme;
+    private final QualifierEncodingScheme qualifierEncodingScheme;
+    private final EncodedCQCounter encodedCQCounter;
+    private final Boolean useStatsForParallelization;
+
+    public static class Builder {
+        private PTableKey key;
+        private PName name;
+        private PName schemaName = PName.EMPTY_NAME;
+        private PName tableName = PName.EMPTY_NAME;
+        private PName tenantId;
+        private PTableType type;
+        private PIndexState state;
+        private long sequenceNumber;
+        private long timeStamp;
+        private long indexDisableTimestamp;
+        private List<PColumn> pkColumns;
+        private List<PColumn> allColumns;
+        private List<PColumn> excludedColumns;
+        private List<PColumnFamily> families;
+        private Map<byte[], PColumnFamily> familyByBytes;
+        private Map<String, PColumnFamily> familyByString;
+        private ListMultimap<String, PColumn> columnsByName;
+        private Map<KVColumnFamilyQualifier, PColumn> kvColumnsByQualifiers;
+        private PName pkName;
+        private Integer bucketNum;
+        private RowKeySchema rowKeySchema;
+        private List<PTable> indexes;
+        private PName parentName;
+        private PName parentSchemaName;
+        private PName parentTableName;
+        private List<PName> physicalNames;
+        private boolean isImmutableRows;
+        private IndexMaintainer indexMaintainer;
+        private ImmutableBytesWritable indexMaintainersPtr;
+        private PName defaultFamilyName;
+        private String viewStatement;
+        private boolean disableWAL;
+        private boolean multiTenant;
+        private boolean storeNulls;
+        private TransactionFactory.Provider transactionProvider;
+        private ViewType viewType;
+        private PDataType viewIndexType;
+        private Long viewIndexId;
+        private int estimatedSize;
+        private IndexType indexType;
+        private int baseColumnCount;
+        private boolean rowKeyOrderOptimizable;
+        private boolean hasColumnsRequiringUpgrade;
+        private int rowTimestampColPos;
+        private long updateCacheFrequency;
+        private boolean isNamespaceMapped;
+        private String autoPartitionSeqName;
+        private boolean isAppendOnlySchema;
+        private ImmutableStorageScheme immutableStorageScheme;
+        private QualifierEncodingScheme qualifierEncodingScheme;
+        private EncodedCQCounter encodedCQCounter;
+        private Boolean useStatsForParallelization;
+        // Optionally set columns for the builder, but not for the actual PTable
+        private Collection<PColumn> columns;
+
+        public Builder setKey(PTableKey key) {
+            this.key = key;
+            return this;
+        }
+
+        public Builder setName(PName name) {
+            this.name = name;
+            return this;
+        }
+
+        public Builder setSchemaName(PName schemaName) {
+            this.schemaName = schemaName;
+            return this;
+        }
+
+        public Builder setTableName(PName tableName) {
+            this.tableName = tableName;
+            return this;
+        }
+
+        public Builder setTenantId(PName tenantId) {
+            this.tenantId = tenantId;
+            return this;
+        }
+
+        public Builder setType(PTableType type) {
+            this.type = type;
+            return this;
+        }
+
+        public Builder setState(PIndexState state) {
+            this.state = state;
+            return this;
+        }
+
+        public Builder setSequenceNumber(long sequenceNumber) {
+            this.sequenceNumber = sequenceNumber;
+            return this;
+        }
+
+        public Builder setTimeStamp(long timeStamp) {
+            this.timeStamp = timeStamp;
+            return this;
+        }
+
+        public Builder setIndexDisableTimestamp(long indexDisableTimestamp) {
+            this.indexDisableTimestamp = indexDisableTimestamp;
+            return this;
+        }
+
+        public Builder setPkColumns(List<PColumn> pkColumns) {
+            this.pkColumns = pkColumns;
+            return this;
+        }
+
+        public Builder setAllColumns(List<PColumn> allColumns) {
+            this.allColumns = allColumns;
+            return this;
+        }
+
+        public Builder setExcludedColumns(List<PColumn> excludedColumns) {
+            this.excludedColumns = excludedColumns;
+            return this;
+        }
+
+        public Builder setFamilyAttributes(List<PColumnFamily> families) {
+            this.familyByBytes = Maps.newHashMapWithExpectedSize(families.size());
+            this.familyByString = Maps.newHashMapWithExpectedSize(families.size());
+            for (PColumnFamily family : families) {
+                familyByBytes.put(family.getName().getBytes(), family);
+                familyByString.put(family.getName().getString(), family);
+            }
+            this.families = families;
+            return this;
+        }
+
+        public Builder setFamilies(List<PColumnFamily> families) {
+            this.families = families;
+            return this;
+        }
+
+        public Builder setFamilyByBytes(Map<byte[], PColumnFamily> familyByBytes) {
+            this.familyByBytes = familyByBytes;
+            return this;
+        }
+
+        public Builder setFamilyByString(Map<String, PColumnFamily> familyByString) {
+            this.familyByString = familyByString;
+            return this;
+        }
+
+        public Builder setColumnsByName(ListMultimap<String, PColumn> columnsByName) {
+            this.columnsByName = columnsByName;
+            return this;
+        }
+
+        public Builder setKvColumnsByQualifiers(Map<KVColumnFamilyQualifier, PColumn> kvColumnsByQualifiers) {
+            this.kvColumnsByQualifiers = kvColumnsByQualifiers;
+            return this;
+        }
+
+        public Builder setPkName(PName pkName) {
+            this.pkName = pkName;
+            return this;
+        }
+
+        public Builder setBucketNum(Integer bucketNum) {
+            this.bucketNum = bucketNum;
+            return this;
+        }
+
+        public Builder setRowKeySchema(RowKeySchema rowKeySchema) {
+            this.rowKeySchema = rowKeySchema;
+            return this;
+        }
+
+        public Builder setIndexes(List<PTable> indexes) {
+            this.indexes = indexes;
+            return this;
+        }
+
+        public Builder setParentName(PName parentName) {
+            this.parentName = parentName;
+            return this;
+        }
+
+        public Builder setParentSchemaName(PName parentSchemaName) {
+            this.parentSchemaName = parentSchemaName;
+            return this;
+        }
+
+        public Builder setParentTableName(PName parentTableName) {
+            this.parentTableName = parentTableName;
+            return this;
+        }
+
+        public Builder setPhysicalNames(List<PName> physicalNames) {
+            this.physicalNames = physicalNames;
+            return this;
+        }
+
+        public Builder setImmutableRows(boolean immutableRows) {
+            isImmutableRows = immutableRows;
+            return this;
+        }
+
+        public Builder setIndexMaintainer(IndexMaintainer indexMaintainer) {
+            this.indexMaintainer = indexMaintainer;
+            return this;
+        }
+
+        public Builder setIndexMaintainersPtr(ImmutableBytesWritable indexMaintainersPtr) {
+            this.indexMaintainersPtr = indexMaintainersPtr;
+            return this;
+        }
+
+        public Builder setDefaultFamilyName(PName defaultFamilyName) {
+            this.defaultFamilyName = defaultFamilyName;
+            return this;
+        }
+
+        public Builder setViewStatement(String viewStatement) {
+            this.viewStatement = viewStatement;
+            return this;
+        }
+
+        public Builder setDisableWAL(boolean disableWAL) {
+            this.disableWAL = disableWAL;
+            return this;
+        }
+
+        public Builder setMultiTenant(boolean multiTenant) {
+            this.multiTenant = multiTenant;
+            return this;
+        }
+
+        public Builder setStoreNulls(boolean storeNulls) {
+            this.storeNulls = storeNulls;
+            return this;
+        }
+
+        public Builder setTransactionProvider(TransactionFactory.Provider transactionProvider) {
+            this.transactionProvider = transactionProvider;
+            return this;
+        }
+
+        public Builder setViewType(ViewType viewType) {
+            this.viewType = viewType;
+            return this;
+        }
+
+        public Builder setViewIndexType(PDataType viewIndexType) {
+            this.viewIndexType = viewIndexType;
+            return this;
+        }
+
+        public Builder setViewIndexId(Long viewIndexId) {
+            this.viewIndexId = viewIndexId;
+            return this;
+        }
+
+        public Builder setEstimatedSize(int estimatedSize) {
+            this.estimatedSize = estimatedSize;
+            return this;
+        }
+
+        public Builder setIndexType(IndexType indexType) {
+            this.indexType = indexType;
+            return this;
+        }
+
+        public Builder setBaseColumnCount(int baseColumnCount) {
+            this.baseColumnCount = baseColumnCount;
+            return this;
+        }
+
+        public Builder setRowKeyOrderOptimizable(boolean rowKeyOrderOptimizable) {
+            this.rowKeyOrderOptimizable = rowKeyOrderOptimizable;
+            return this;
+        }
+
+        public Builder setHasColumnsRequiringUpgrade(boolean hasColumnsRequiringUpgrade) {
+            this.hasColumnsRequiringUpgrade = hasColumnsRequiringUpgrade;
+            return this;
+        }
+
+        public Builder setRowTimestampColPos(int rowTimestampColPos) {
+            this.rowTimestampColPos = rowTimestampColPos;
+            return this;
+        }
+
+        public Builder setUpdateCacheFrequency(long updateCacheFrequency) {
+            this.updateCacheFrequency = updateCacheFrequency;
+            return this;
+        }
+
+        public Builder setNamespaceMapped(boolean namespaceMapped) {
+            isNamespaceMapped = namespaceMapped;
+            return this;
+        }
+
+        public Builder setAutoPartitionSeqName(String autoPartitionSeqName) {
+            this.autoPartitionSeqName = autoPartitionSeqName;
+            return this;
+        }
+
+        public Builder setAppendOnlySchema(boolean appendOnlySchema) {
+            isAppendOnlySchema = appendOnlySchema;
+            return this;
+        }
+
+        public Builder setImmutableStorageScheme(ImmutableStorageScheme immutableStorageScheme) {
+            this.immutableStorageScheme = immutableStorageScheme;
+            return this;
+        }
+
+        public Builder setQualifierEncodingScheme(QualifierEncodingScheme qualifierEncodingScheme) {
+            this.qualifierEncodingScheme = qualifierEncodingScheme;
+            return this;
+        }
+
+        public Builder setEncodedCQCounter(EncodedCQCounter encodedCQCounter) {
+            this.encodedCQCounter = encodedCQCounter;
+            return this;
+        }
+
+        public Builder setUseStatsForParallelization(Boolean useStatsForParallelization) {
+            this.useStatsForParallelization = useStatsForParallelization;
+            return this;
+        }
+
+        /**
+         * Note: When set in the builder, we must call {@link Builder#initDerivedAttributes()}
+         * before building the PTable in order to correctly populate other attributes of the PTable
+         * @param columns PColumns to be set in the builder
+         * @return PTableImpl.Builder object
+         */
+        public Builder setColumns(Collection<PColumn> columns) {
+            this.columns = columns;
+            return this;
+        }
+
+        /**
+         * Populate derivable attributes of the PTable
+         * @return PTableImpl.Builder object
+         * @throws SQLException
+         */
+        private Builder initDerivedAttributes() throws SQLException {
+            checkTenantId(this.tenantId);
+            Preconditions.checkNotNull(this.schemaName);
+            Preconditions.checkNotNull(this.tableName);
+            Preconditions.checkNotNull(this.columns);
+            Preconditions.checkNotNull(this.indexes);
+            Preconditions.checkNotNull(this.physicalNames);
+            Preconditions.checkNotNull(this.hasColumnsRequiringUpgrade);
+            Preconditions.checkNotNull(this.rowKeyOrderOptimizable);
+
+            PName fullName = PNameFactory.newName(SchemaUtil.getTableName(
+                    this.schemaName.getString(), this.tableName.getString()));
+            int estimatedSize = SizedUtil.OBJECT_SIZE * 2 + 23 * SizedUtil.POINTER_SIZE +
+                    4 * SizedUtil.INT_SIZE + 2 * SizedUtil.LONG_SIZE + 2 * SizedUtil.INT_OBJECT_SIZE +
+                    PNameFactory.getEstimatedSize(this.tenantId) +
+                    PNameFactory.getEstimatedSize(this.schemaName) +
+                    PNameFactory.getEstimatedSize(this.tableName) +
+                    PNameFactory.getEstimatedSize(this.pkName) +
+                    PNameFactory.getEstimatedSize(this.parentTableName) +
+                    PNameFactory.getEstimatedSize(this.defaultFamilyName);
+            int numPKColumns = 0;
+            List<PColumn> pkColumns;
+            PColumn[] allColumns;
+            if (this.bucketNum != null) {
+                // Add salt column to allColumns and pkColumns, but don't add to
+                // columnsByName, since it should not be addressable via name.
+                allColumns = new PColumn[this.columns.size()+1];
+                allColumns[SALTING_COLUMN.getPosition()] = SALTING_COLUMN;
+                pkColumns = Lists.newArrayListWithExpectedSize(this.columns.size()+1);
+                ++numPKColumns;
+            } else {
+                allColumns = new PColumn[this.columns.size()];
+                pkColumns = Lists.newArrayListWithExpectedSize(this.columns.size());
+            }
+            // Must do this as with the new method of storing diffs, we just care about
+            // ordinal position relative order and not the true ordinal value itself.
+            List<PColumn> sortedColumns = Lists.newArrayList(this.columns);
+            Collections.sort(sortedColumns, new Comparator<PColumn>() {
+                @Override
+                public int compare(PColumn o1, PColumn o2) {
+                    return Integer.valueOf(o1.getPosition()).compareTo(o2.getPosition());
+                }
+            });
+
+            int position = 0;
+            if (this.bucketNum != null) {
+                position = 1;
+            }
+            ListMultimap<String, PColumn> populateColumnsByName =
+                    ArrayListMultimap.create(this.columns.size(), 1);
+            Map<KVColumnFamilyQualifier, PColumn> populateKvColumnsByQualifiers =
+                    Maps.newHashMapWithExpectedSize(this.columns.size());
+            for (PColumn column : sortedColumns) {
+                allColumns[position] = column;
+                position++;
+                PName familyName = column.getFamilyName();
+                if (familyName == null) {
+                    ++numPKColumns;
+                }
+                String columnName = column.getName().getString();
+                if (populateColumnsByName.put(columnName, column)) {
+                    int count = 0;
+                    for (PColumn dupColumn : populateColumnsByName.get(columnName)) {
+                        if (Objects.equal(familyName, dupColumn.getFamilyName())) {
+                            count++;
+                            if (count > 1) {
+                                throw new ColumnAlreadyExistsException(this.schemaName.getString(),
+                                        fullName.getString(), columnName);
+                            }
+                        }
+                    }
+                }
+                byte[] cq = column.getColumnQualifierBytes();
+                String cf = column.getFamilyName() != null ?
+                        column.getFamilyName().getString() : null;
+                if (cf != null && cq != null) {
+                    KVColumnFamilyQualifier info = new KVColumnFamilyQualifier(cf, cq);
+                    if (populateKvColumnsByQualifiers.get(info) != null) {
+                        throw new ColumnAlreadyExistsException(this.schemaName.getString(),
+                                fullName.getString(), columnName);
+                    }
+                    populateKvColumnsByQualifiers.put(info, column);
+                }
+            }
+            estimatedSize += SizedUtil.sizeOfMap(allColumns.length, SizedUtil.POINTER_SIZE,
+                    SizedUtil.sizeOfArrayList(1)); // for multi-map
+            estimatedSize += SizedUtil.sizeOfMap(numPKColumns) +
+                    SizedUtil.sizeOfMap(allColumns.length);
+
+            RowKeySchemaBuilder builder = new RowKeySchemaBuilder(numPKColumns);
+            // Two pass so that column order in column families matches overall column order
+            // and to ensure that column family order is constant
+            int maxExpectedSize = allColumns.length - numPKColumns;
+            // Maintain iteration order so that column families are ordered as they are listed
+            Map<PName, List<PColumn>> familyMap = Maps.newLinkedHashMap();
+            PColumn rowTimestampCol = null;
+            boolean hasColsRequiringUpgrade = false;
+            for (PColumn column : allColumns) {
+                PName familyName = column.getFamilyName();
+                if (familyName == null) {
+                    hasColsRequiringUpgrade |=
+                            (column.getSortOrder() == SortOrder.DESC
+                                    && (!column.getDataType().isFixedWidth()
+                                    || column.getDataType() == PChar.INSTANCE
+                                    || column.getDataType() == PFloat.INSTANCE
+                                    || column.getDataType() == PDouble.INSTANCE
+                                    || column.getDataType() == PBinary.INSTANCE) )
+                                    || (column.getSortOrder() == SortOrder.ASC
+                                        && column.getDataType() == PBinary.INSTANCE
+                                        && column.getMaxLength() != null
+                                        && column.getMaxLength() > 1);
+                    pkColumns.add(column);
+                    if (column.isRowTimestamp()) {
+                        rowTimestampCol = column;
+                    }
+                }
+                if (familyName == null) {
+                    estimatedSize += column.getEstimatedSize(); // PK columns
+                    builder.addField(column, column.isNullable(), column.getSortOrder());
+                } else {
+                    List<PColumn> columnsInFamily = familyMap.get(familyName);
+                    if (columnsInFamily == null) {
+                        columnsInFamily = Lists.newArrayListWithExpectedSize(maxExpectedSize);
+                        familyMap.put(familyName, columnsInFamily);
+                    }
+                    columnsInFamily.add(column);
+                }
+            }
+            int rowTimestampColPos;
+            if (rowTimestampCol != null) {
+                rowTimestampColPos = pkColumns.indexOf(rowTimestampCol);
+            } else {
+                rowTimestampColPos = -1;
+            }
+
+            Iterator<Map.Entry<PName,List<PColumn>>> iterator = familyMap.entrySet().iterator();
+            PColumnFamily[] families = new PColumnFamily[familyMap.size()];
+            ImmutableMap.Builder<String, PColumnFamily> familyByString = ImmutableMap.builder();
+            ImmutableSortedMap.Builder<byte[], PColumnFamily> familyByBytes = ImmutableSortedMap
+                    .orderedBy(Bytes.BYTES_COMPARATOR);
+            for (int i = 0; i < families.length; i++) {
+                Map.Entry<PName,List<PColumn>> entry = iterator.next();
+                PColumnFamily family = new PColumnFamilyImpl(entry.getKey(), entry.getValue());
+                families[i] = family;
+                familyByString.put(family.getName().getString(), family);
+                familyByBytes.put(family.getName().getBytes(), family);
+                estimatedSize += family.getEstimatedSize();
+            }
+            estimatedSize += SizedUtil.sizeOfArrayList(families.length);
+            estimatedSize += SizedUtil.sizeOfMap(families.length) * 2;
+            for (PTable index : this.indexes) {
+                estimatedSize += index.getEstimatedSize();
+            }
+
+            estimatedSize += PNameFactory.getEstimatedSize(this.parentName);
+            for (PName physicalName : this.physicalNames) {
+                estimatedSize += physicalName.getEstimatedSize();
+            }
+            // Populate the derived fields and return the builder
+            return this.setName(fullName)
+                    .setKey(new PTableKey(this.tenantId, fullName.getString()))
+                    .setParentName(this.parentTableName == null ? null :
+                            PNameFactory.newName(SchemaUtil.getTableName(
+                                    this.parentSchemaName != null ?
+                                            this.parentSchemaName.getString() : null,
+                                    this.parentTableName.getString())))
+                    .setColumnsByName(populateColumnsByName)
+                    .setKvColumnsByQualifiers(populateKvColumnsByQualifiers)
+                    .setAllColumns(ImmutableList.copyOf(allColumns))
+                    .setHasColumnsRequiringUpgrade(hasColsRequiringUpgrade
+                            | this.hasColumnsRequiringUpgrade)
+                    .setPkColumns(ImmutableList.copyOf(pkColumns))
+                    .setRowTimestampColPos(rowTimestampColPos)
+                    // after hasDescVarLengthColumns is calculated
+                    .setRowKeySchema(builder.rowKeyOrderOptimizable(
+                            this.rowKeyOrderOptimizable || !this.hasColumnsRequiringUpgrade)
+                            .build())
+                    .setFamilies(ImmutableList.copyOf(families))
+                    .setFamilyByBytes(familyByBytes.build())
+                    .setFamilyByString(familyByString.build())
+                    .setEstimatedSize(estimatedSize + this.rowKeySchema.getEstimatedSize());
         }
-        this.timeStamp = timeStamp;
-        this.pkColumns = this.allColumns = Collections.emptyList();
-        this.families = Collections.emptyList();
-        this.familyByBytes = Collections.emptyMap();
-        this.familyByString = Collections.emptyMap();
-        this.rowKeySchema = RowKeySchema.EMPTY_SCHEMA;
-        this.indexes = Collections.emptyList();
-        this.physicalNames = Collections.emptyList();;
+
+        public PTableImpl build() throws SQLException {
+            // Note that we call initDerivedAttributes to populate derivable attributes if
+            // this.columns is set in the PTableImpl.Builder object
+            return (this.columns == null) ? new PTableImpl(this) :
+                    new PTableImpl(this.initDerivedAttributes());
+        }
+
+    }
+
+    @VisibleForTesting
+    PTableImpl() {
+        this(new PTableImpl.Builder()
+                .setIndexes(Collections.<PTable>emptyList())
+                .setPhysicalNames(Collections.<PName>emptyList())
+                .setRowKeySchema(RowKeySchema.EMPTY_SCHEMA));
+    }
+
+    // Private constructor used by the builder
+    private PTableImpl(Builder builder) {
+        this.key = builder.key;
+        this.name = builder.name;
+        this.schemaName = builder.schemaName;
+        this.tableName = builder.tableName;
+        this.tenantId = builder.tenantId;
+        this.type = builder.type;
+        this.state = builder.state;
+        this.sequenceNumber = builder.sequenceNumber;
+        this.timeStamp = builder.timeStamp;
+        this.indexDisableTimestamp = builder.indexDisableTimestamp;
+        this.pkColumns = builder.pkColumns;
+        this.allColumns = builder.allColumns;
+        this.excludedColumns = builder.excludedColumns;
+        this.families = builder.families;
+        this.familyByBytes = builder.familyByBytes;
+        this.familyByString = builder.familyByString;
+        this.columnsByName = builder.columnsByName;
+        this.kvColumnsByQualifiers = builder.kvColumnsByQualifiers;
+        this.pkName = builder.pkName;
+        this.bucketNum = builder.bucketNum;
+        this.rowKeySchema = builder.rowKeySchema;
+        this.indexes = builder.indexes;
+        this.parentName = builder.parentName;
+        this.parentSchemaName = builder.parentSchemaName;
+        this.parentTableName = builder.parentTableName;
+        this.physicalNames = builder.physicalNames;
+        this.isImmutableRows = builder.isImmutableRows;
+        this.indexMaintainer = builder.indexMaintainer;
+        this.indexMaintainersPtr = builder.indexMaintainersPtr;
+        this.defaultFamilyName = builder.defaultFamilyName;
+        this.viewStatement = builder.viewStatement;
+        this.disableWAL = builder.disableWAL;
+        this.multiTenant = builder.multiTenant;
+        this.storeNulls = builder.storeNulls;
+        this.transactionProvider = builder.transactionProvider;
+        this.viewType = builder.viewType;
+        this.viewIndexType = builder.viewIndexType;
+        this.viewIndexId = builder.viewIndexId;
+        this.estimatedSize = builder.estimatedSize;
+        this.indexType = builder.indexType;
+        this.baseColumnCount = builder.baseColumnCount;
+        this.rowKeyOrderOptimizable = builder.rowKeyOrderOptimizable;
+        this.hasColumnsRequiringUpgrade = builder.hasColumnsRequiringUpgrade;
+        this.rowTimestampColPos = builder.rowTimestampColPos;
+        this.updateCacheFrequency = builder.updateCacheFrequency;
+        this.isNamespaceMapped = builder.isNamespaceMapped;
+        this.autoPartitionSeqName = builder.autoPartitionSeqName;
+        this.isAppendOnlySchema = builder.isAppendOnlySchema;
+        this.immutableStorageScheme = builder.immutableStorageScheme;
+        this.qualifierEncodingScheme = builder.qualifierEncodingScheme;
+        this.encodedCQCounter = builder.encodedCQCounter;
+        this.useStatsForParallelization = builder.useStatsForParallelization;
     }
 
     // When cloning table, ignore the salt column as it will be added back in the constructor
@@ -267,153 +764,65 @@ public class PTableImpl implements PTable {
         return table.getBucketNum() == null ? table.getColumns() : table.getColumns().subList(1, table.getColumns().size());
     }
 
-    public static PTableImpl makePTable(PTable table, long timeStamp, List<PTable> indexes) throws SQLException {
-        return new PTableImpl(table, table.rowKeyOrderOptimizable(), table.getIndexState(), timeStamp,
-                table.getSequenceNumber(), getColumnsToClone(table), table.getDefaultFamilyName(), table.getType(),
-                table.getBaseColumnCount(), table.getSchemaName(), table.getTableName(), table.getViewStatement(),
-                table.getUpdateCacheFrequency(), table.getTenantId(), indexes);
-    }
-
-    public static PTable makePTable(PTable index, PName indexName, String viewStatement, long updateCacheFrequency,
-            PName tenantId) throws SQLException {
-        return Objects.equal(viewStatement, index.getViewStatement()) ? index
-                : new PTableImpl(index, index.rowKeyOrderOptimizable(), index.getIndexState(), index.getTimeStamp(),
-                        index.getSequenceNumber(), index.getColumns(), index.getDefaultFamilyName(), index.getType(),
-                        index.getBaseColumnCount(), index.getSchemaName(), indexName,
-                        viewStatement, updateCacheFrequency, tenantId,
-                        index.getIndexes());
-    }
-    
-    public static PTableImpl makePTable(PTable table, Collection<PColumn> columns) throws SQLException {
-        return new PTableImpl(table, table.rowKeyOrderOptimizable(), table.getIndexState(), table.getTimeStamp(),
-                table.getSequenceNumber(), columns, table.getDefaultFamilyName(), table.getType(),
-                table.getBaseColumnCount(), table.getSchemaName(), table.getTableName(), table.getViewStatement(),
-                table.getUpdateCacheFrequency(), table.getTenantId(), table.getIndexes());
+    /**
+     * Get a PTableImpl.Builder from an existing PTable and set the builder columns
+     * @param table Original PTable
+     * @param columns Columns to set in the builder for the new PTable to be constructed
+     * @return PTable builder object based on an existing PTable
+     */
+    public static PTableImpl.Builder builderWithColumns(PTable table, Collection<PColumn> columns) {
+        return builderFromExisting(table).setColumns(columns);
     }
 
     /**
-     * Used to create a PTable for views or view indexes, the basePTable is for attributes we inherit from the physical table
+     * Get a PTableImpl.Builder from an existing PTable
+     * @param table Original PTable
      */
-    public static PTableImpl makePTable(PTable view, PTable baseTable, Collection<PColumn> columns, long timestamp, int baseTableColumnCount, Collection<PColumn> excludedColumns) throws SQLException {
-        // if a TableProperty is not valid on a view we set it to the base table value
-        // if a TableProperty is valid on a view and is not mutable on a view we set it to the base table value
-        // if a TableProperty is valid on a view and is mutable on a view we use the value set on the view 
-        return new PTableImpl(
-            view.getTenantId(), view.getSchemaName(), view.getTableName(), view.getType(), view.getIndexState(), timestamp,
-            view.getSequenceNumber(), view.getPKName(), view.getBucketNum(), columns, view.getParentSchemaName(), view.getParentTableName(),
-            view.getIndexes(), baseTable.isImmutableRows(), view.getPhysicalNames(), view.getDefaultFamilyName(), view.getViewStatement(),
-            baseTable.isWALDisabled(), baseTable.isMultiTenant(), baseTable.getStoreNulls(), view.getViewType(),
-            view.getViewIndexType(), view.getViewIndexId(), view.getIndexType(),
-            baseTableColumnCount, view.rowKeyOrderOptimizable(), baseTable.getTransactionProvider(), view.getUpdateCacheFrequency(),
-            view.getIndexDisableTimestamp(), view.isNamespaceMapped(), baseTable.getAutoPartitionSeqName(), baseTable.isAppendOnlySchema(),
-            baseTable.getImmutableStorageScheme(), baseTable.getEncodingScheme(), view.getEncodedCQCounter(), view.useStatsForParallelization(), excludedColumns);
+    private static PTableImpl.Builder builderFromExisting(PTable table) {
+        return new PTableImpl.Builder()
+                .setType(table.getType())
+                .setState(table.getIndexState())
+                .setTimeStamp(table.getTimeStamp())
+                .setIndexDisableTimestamp(table.getIndexDisableTimestamp())
+                .setSequenceNumber(table.getSequenceNumber())
+                .setImmutableRows(table.isImmutableRows())
+                .setViewStatement(table.getViewStatement())
+                .setDisableWAL(table.isWALDisabled())
+                .setMultiTenant(table.isMultiTenant())
+                .setStoreNulls(table.getStoreNulls())
+                .setViewType(table.getViewType())
+                .setViewIndexType(table.getViewIndexType())
+                .setViewIndexId(table.getViewIndexId())
+                .setIndexType(table.getIndexType())
+                .setTransactionProvider(table.getTransactionProvider())
+                .setUpdateCacheFrequency(table.getUpdateCacheFrequency())
+                .setNamespaceMapped(table.isNamespaceMapped())
+                .setAutoPartitionSeqName(table.getAutoPartitionSeqName())
+                .setAppendOnlySchema(table.isAppendOnlySchema())
+                .setImmutableStorageScheme(table.getImmutableStorageScheme() == null ?
+                        ImmutableStorageScheme.ONE_CELL_PER_COLUMN : table.getImmutableStorageScheme())
+                .setQualifierEncodingScheme(table.getEncodingScheme() == null ?
+                        QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : table.getEncodingScheme())
+                .setBaseColumnCount(table.getBaseColumnCount())
+                .setEncodedCQCounter(table.getEncodedCQCounter())
+                .setUseStatsForParallelization(table.useStatsForParallelization())
+                .setExcludedColumns(table.getExcludedColumns() == null ?
+                        ImmutableList.<PColumn>of() : ImmutableList.copyOf(table.getExcludedColumns()))
+                .setTenantId(table.getTenantId())
+                .setSchemaName(table.getSchemaName())
+                .setTableName(table.getTableName())
+                .setPkName(table.getPKName())
+                .setDefaultFamilyName(table.getDefaultFamilyName())
+                .setRowKeyOrderOptimizable(table.rowKeyOrderOptimizable())
+                .setBucketNum(table.getBucketNum())
+                .setIndexes(table.getIndexes() == null ?
+                        Collections.<PTable>emptyList() : table.getIndexes())
+                .setParentSchemaName(table.getParentSchemaName())
+                .setParentTableName(table.getParentTableName())
+                .setPhysicalNames(table.getPhysicalNames() == null ?
+                        ImmutableList.<PName>of() : ImmutableList.copyOf(table.getPhysicalNames()));
     }
-    
-    public static PTableImpl makePTable(PTable table, PTableType type, Collection<PColumn> columns) throws SQLException {
-        return new PTableImpl(table, table.rowKeyOrderOptimizable(), table.getIndexState(), table.getTimeStamp(),
-                table.getSequenceNumber(), columns, table.getDefaultFamilyName(), type,
-                table.getBaseColumnCount(), table.getSchemaName(), table.getTableName(), table.getViewStatement(),
-                table.getUpdateCacheFrequency(), table.getTenantId(), table.getIndexes());
-    }
-
-    public static PTableImpl makePTable(PTable table, Collection<PColumn> columns, PName defaultFamily)
-            throws SQLException {
-        return new PTableImpl(table, table.rowKeyOrderOptimizable(), table.getIndexState(), table.getTimeStamp(),
-                table.getSequenceNumber(), columns, defaultFamily, table.getType(),
-                table.getBaseColumnCount(), table.getSchemaName(), table.getTableName(), table.getViewStatement(),
-                table.getUpdateCacheFrequency(), table.getTenantId(), table.getIndexes());
-    }
-
-    public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, Collection<PColumn> columns)
-            throws SQLException {
-        return new PTableImpl(table, table.rowKeyOrderOptimizable(), table.getIndexState(), timeStamp,
-                sequenceNumber, columns, table.getDefaultFamilyName(), table.getType(),
-                table.getBaseColumnCount(), table.getSchemaName(), table.getTableName(), table.getViewStatement(),
-                table.getUpdateCacheFrequency(), table.getTenantId(), table.getIndexes());
-    }
-
-    public static PTableImpl makePTable(PTable table, PIndexState state) throws SQLException {
-        return new PTableImpl(table, table.rowKeyOrderOptimizable(), state, table.getTimeStamp(),
-                table.getSequenceNumber(), getColumnsToClone(table), table.getDefaultFamilyName(), table.getType(),
-                table.getBaseColumnCount(), table.getSchemaName(), table.getTableName(), table.getViewStatement(),
-                table.getUpdateCacheFrequency(), table.getTenantId(), table.getIndexes());
-    }
-
-    public static PTableImpl makePTable(PTable table, boolean rowKeyOrderOptimizable) throws SQLException {
-        return new PTableImpl(table, rowKeyOrderOptimizable, table.getIndexState(), table.getTimeStamp(),
-                table.getSequenceNumber(), getColumnsToClone(table), table.getDefaultFamilyName(), table.getType(),
-                table.getBaseColumnCount(), table.getSchemaName(), table.getTableName(), table.getViewStatement(),
-                table.getUpdateCacheFrequency(), table.getTenantId(), table.getIndexes());
-    }
-
-    public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type,
-            PIndexState state, long timeStamp, long sequenceNumber, PName pkName, Integer bucketNum,
-            Collection<PColumn> columns, PName dataSchemaName, PName dataTableName, List<PTable> indexes,
-            boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression,
-            boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, PDataType viewIndexType, Long viewIndexId,
-            IndexType indexType, boolean rowKeyOrderOptimizable, TransactionFactory.Provider transactionProvider,
-            long updateCacheFrequency, long indexDisableTimestamp, boolean isNamespaceMapped,
-            String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme,
-            QualifierEncodingScheme qualifierEncodingScheme, EncodedCQCounter encodedCQCounter,
-            Boolean useStatsForParallelization) throws SQLException {
-        return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName,
-                bucketNum, columns, dataSchemaName, dataTableName, indexes, isImmutableRows, physicalNames,
-                defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexType, viewIndexId,
-                indexType, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, rowKeyOrderOptimizable, transactionProvider,
-                updateCacheFrequency, indexDisableTimestamp, isNamespaceMapped, autoPartitionSeqName,
-                isAppendOnlySchema, storageScheme, qualifierEncodingScheme, encodedCQCounter,
-                useStatsForParallelization, null);
-    }
-
-    public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type,
-            PIndexState state, long timeStamp, long sequenceNumber, PName pkName, Integer bucketNum,
-            Collection<PColumn> columns, PName dataSchemaName, PName dataTableName, List<PTable> indexes,
-            boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression,
-            boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, PDataType viewIndexType, Long viewIndexId,
-            IndexType indexType, boolean rowKeyOrderOptimizable, TransactionFactory.Provider transactionProvider,
-            long updateCacheFrequency, int baseColumnCount, long indexDisableTimestamp, boolean isNamespaceMapped,
-            String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme,
-            QualifierEncodingScheme qualifierEncodingScheme, EncodedCQCounter encodedCQCounter,
-            Boolean useStatsForParallelization) throws SQLException {
-        return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName,
-                bucketNum, columns, dataSchemaName, dataTableName, indexes, isImmutableRows, physicalNames,
-                defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType,viewIndexType,  viewIndexId,
-                indexType, baseColumnCount, rowKeyOrderOptimizable, transactionProvider, updateCacheFrequency,
-                indexDisableTimestamp, isNamespaceMapped, autoPartitionSeqName, isAppendOnlySchema, storageScheme,
-                qualifierEncodingScheme, encodedCQCounter, useStatsForParallelization, null);
-    }
-
-    private PTableImpl(PTable table, boolean rowKeyOrderOptimizable, PIndexState state, long timeStamp,
-            long sequenceNumber, Collection<PColumn> columns, PName defaultFamily, PTableType type,
-            int baseTableColumnCount, PName schemaName, PName tableName, String viewStatement,
-            long updateCacheFrequency, PName tenantId, List<PTable> indexes) throws SQLException {
-        init(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, table.getPKName(),
-                table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), indexes,
-                table.isImmutableRows(), table.getPhysicalNames(), defaultFamily, viewStatement, table.isWALDisabled(),
-                table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexType(), table.getViewIndexId(),
-                table.getIndexType(), baseTableColumnCount, rowKeyOrderOptimizable, table.getTransactionProvider(),
-                updateCacheFrequency, table.getIndexDisableTimestamp(), table.isNamespaceMapped(),
-                table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(),
-                table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization(), null);
-    }
-
-    private PTableImpl(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state,
-            long timeStamp, long sequenceNumber, PName pkName, Integer bucketNum, Collection<PColumn> columns,
-            PName parentSchemaName, PName parentTableName, List<PTable> indexes, boolean isImmutableRows,
-            List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant,
-            boolean storeNulls, ViewType viewType, PDataType viewIndexType, Long viewIndexId, IndexType indexType,
-            int baseColumnCount, boolean rowKeyOrderOptimizable, TransactionFactory.Provider transactionProvider, long updateCacheFrequency,
-            long indexDisableTimestamp, boolean isNamespaceMapped, String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme, 
-            QualifierEncodingScheme qualifierEncodingScheme, EncodedCQCounter encodedCQCounter,
-            Boolean useStatsForParallelization, Collection<PColumn> excludedColumns)
-            throws SQLException {
-        init(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns,
-                parentSchemaName, parentTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName,
-                viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable,
-                transactionProvider, updateCacheFrequency, indexDisableTimestamp, isNamespaceMapped, autoPartitionSeqName, isAppendOnlySchema, storageScheme, 
-                qualifierEncodingScheme, encodedCQCounter, useStatsForParallelization, excludedColumns);
-    }
-    
+
     @Override
     public long getUpdateCacheFrequency() {
         return updateCacheFrequency;
@@ -434,208 +843,14 @@ public class PTableImpl implements PTable {
         return viewType;
     }
 
-
     @Override
     public int getEstimatedSize() {
         return estimatedSize;
     }
 
-    private void init(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, long timeStamp, long sequenceNumber,
-            PName pkName, Integer bucketNum, Collection<PColumn> columns, PName parentSchemaName, PName parentTableName,
-            List<PTable> indexes, boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL,
-            boolean multiTenant, boolean storeNulls, ViewType viewType,PDataType viewIndexType,  Long viewIndexId,
-            IndexType indexType , int baseColumnCount, boolean rowKeyOrderOptimizable, TransactionFactory.Provider transactionProvider, long updateCacheFrequency, long indexDisableTimestamp, 
-            boolean isNamespaceMapped, String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme, QualifierEncodingScheme qualifierEncodingScheme, 
-            EncodedCQCounter encodedCQCounter, Boolean useStatsForParallelization, Collection<PColumn> excludedColumns) throws SQLException {
-        Preconditions.checkNotNull(schemaName);
-        Preconditions.checkArgument(tenantId==null || tenantId.getBytes().length > 0); // tenantId should be null or not empty
-        int estimatedSize = SizedUtil.OBJECT_SIZE * 2 + 23 * SizedUtil.POINTER_SIZE + 4 * SizedUtil.INT_SIZE + 2 * SizedUtil.LONG_SIZE + 2 * SizedUtil.INT_OBJECT_SIZE +
-              PNameFactory.getEstimatedSize(tenantId) +
-              PNameFactory.getEstimatedSize(schemaName) +
-              PNameFactory.getEstimatedSize(tableName) +
-              PNameFactory.getEstimatedSize(pkName) +
-              PNameFactory.getEstimatedSize(parentTableName) +
-              PNameFactory.getEstimatedSize(defaultFamilyName);
-        this.tenantId = tenantId;
-        this.schemaName = schemaName;
-        this.tableName = tableName;
-        this.name = PNameFactory.newName(SchemaUtil.getTableName(schemaName.getString(), tableName.getString()));
-        this.key = new PTableKey(tenantId, name.getString());
-        this.type = type;
-        this.state = state;
-        this.timeStamp = timeStamp;
-        this.indexDisableTimestamp = indexDisableTimestamp;
-        this.sequenceNumber = sequenceNumber;
-        this.pkName = pkName;
-        this.isImmutableRows = isImmutableRows;
-        this.defaultFamilyName = defaultFamilyName;
-        this.viewStatement = viewExpression;
-        this.disableWAL = disableWAL;
-        this.multiTenant = multiTenant;
-        this.storeNulls = storeNulls;
-        this.viewType = viewType;
-        this.viewIndexType = viewIndexType;
-        this.viewIndexId = viewIndexId;
-        this.indexType = indexType;
-        this.transactionProvider = transactionProvider;
-        this.rowKeyOrderOptimizable = rowKeyOrderOptimizable;
-        this.updateCacheFrequency = updateCacheFrequency;
-        this.isNamespaceMapped = isNamespaceMapped;
-        this.autoPartitionSeqName = autoPartitionSeqName;
-        this.isAppendOnlySchema = isAppendOnlySchema;
-        // null check for backward compatibility and sanity. If any of the two below is null, then it means the table is a non-encoded table.
-        this.immutableStorageScheme = storageScheme == null ? ImmutableStorageScheme.ONE_CELL_PER_COLUMN : storageScheme;
-        this.qualifierEncodingScheme = qualifierEncodingScheme == null ? QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : qualifierEncodingScheme;
-        List<PColumn> pkColumns;
-        PColumn[] allColumns;
-        
-        this.columnsByName = ArrayListMultimap.create(columns.size(), 1);
-        this.kvColumnsByQualifiers = Maps.newHashMapWithExpectedSize(columns.size());
-        int numPKColumns = 0;
-        if (bucketNum != null) {
-            // Add salt column to allColumns and pkColumns, but don't add to
-            // columnsByName, since it should not be addressable via name.
-            allColumns = new PColumn[columns.size()+1];
-            allColumns[SALTING_COLUMN.getPosition()] = SALTING_COLUMN;
-            pkColumns = Lists.newArrayListWithExpectedSize(columns.size()+1);
-            ++numPKColumns;
-        } else {
-            allColumns = new PColumn[columns.size()];
-            pkColumns = Lists.newArrayListWithExpectedSize(columns.size());
-        }
-        // Must do this as with the new method of storing diffs, we just care about ordinal position
-        // relative order and not the true ordinal value itself.
-        List<PColumn> sortedColumns = Lists.newArrayList(columns);
-        Collections.sort(sortedColumns, new Comparator<PColumn>() {
-            @Override
-            public int compare(PColumn o1, PColumn o2) {
-                return Integer.valueOf(o1.getPosition()).compareTo(o2.getPosition());
-            }
-        });
-
-        int position = 0;
-        if (bucketNum != null) {
-            position = 1;
-        }
-        for (PColumn column : sortedColumns) {
-            allColumns[position] = column;
-            position++;
-            PName familyName = column.getFamilyName();
-            if (familyName == null) {
-                ++numPKColumns;
-            }
-            String columnName = column.getName().getString();
-            if (columnsByName.put(columnName, column)) {
-                int count = 0;
-                for (PColumn dupColumn : columnsByName.get(columnName)) {
-                    if (Objects.equal(familyName, dupColumn.getFamilyName())) {
-                        count++;
-                        if (count > 1) {
-                            throw new ColumnAlreadyExistsException(schemaName.getString(), name.getString(), columnName);
-                        }
-                    }
-                }
-            }
-            byte[] cq = column.getColumnQualifierBytes();
-            String cf = column.getFamilyName() != null ? column.getFamilyName().getString() : null;
-            if (cf != null && cq != null) {
-                KVColumnFamilyQualifier info = new KVColumnFamilyQualifier(cf, cq);
-                if (kvColumnsByQualifiers.get(info) != null) {
-                    throw new ColumnAlreadyExistsException(schemaName.getString(),
-                            name.getString(), columnName);
-                }
-                kvColumnsByQualifiers.put(info, column);
-            }
-        }
-        estimatedSize += SizedUtil.sizeOfMap(allColumns.length, SizedUtil.POINTER_SIZE, SizedUtil.sizeOfArrayList(1)); // for multi-map
-
-        this.bucketNum = bucketNum;
-        this.allColumns = ImmutableList.copyOf(allColumns);
-        estimatedSize += SizedUtil.sizeOfMap(numPKColumns) + SizedUtil.sizeOfMap(allColumns.length);
-
-        RowKeySchemaBuilder builder = new RowKeySchemaBuilder(numPKColumns);
-        // Two pass so that column order in column families matches overall column order
-        // and to ensure that column family order is constant
-        int maxExpectedSize = allColumns.length - numPKColumns;
-        // Maintain iteration order so that column families are ordered as they are listed
-        Map<PName, List<PColumn>> familyMap = Maps.newLinkedHashMap();
-        PColumn rowTimestampCol = null;
-        for (PColumn column : allColumns) {
-            PName familyName = column.getFamilyName();
-            if (familyName == null) {
-                hasColumnsRequiringUpgrade |= 
-                        ( column.getSortOrder() == SortOrder.DESC 
-                            && (!column.getDataType().isFixedWidth() 
-                                || column.getDataType() == PChar.INSTANCE 
-                                || column.getDataType() == PFloat.INSTANCE 
-                                || column.getDataType() == PDouble.INSTANCE 
-                                || column.getDataType() == PBinary.INSTANCE) )
-                        || (column.getSortOrder() == SortOrder.ASC && column.getDataType() == PBinary.INSTANCE && column.getMaxLength() != null && column.getMaxLength() > 1);
-                pkColumns.add(column);
-                if (column.isRowTimestamp()) {
-                    rowTimestampCol = column;
-                }
-            }
-            if (familyName == null) {
-                estimatedSize += column.getEstimatedSize(); // PK columns
-                builder.addField(column, column.isNullable(), column.getSortOrder());
-            } else {
-                List<PColumn> columnsInFamily = familyMap.get(familyName);
-                if (columnsInFamily == null) {
-                    columnsInFamily = Lists.newArrayListWithExpectedSize(maxExpectedSize);
-                    familyMap.put(familyName, columnsInFamily);
-                }
-                columnsInFamily.add(column);
-            }
-        }
-        this.pkColumns = ImmutableList.copyOf(pkColumns);
-        if (rowTimestampCol != null) {
-            this.rowTimestampColPos = this.pkColumns.indexOf(rowTimestampCol);
-        } else {
-            this.rowTimestampColPos = -1;
-        }
-        
-        builder.rowKeyOrderOptimizable(this.rowKeyOrderOptimizable()); // after hasDescVarLengthColumns is calculated
-        this.rowKeySchema = builder.build();
-        estimatedSize += rowKeySchema.getEstimatedSize();
-        Iterator<Map.Entry<PName,List<PColumn>>> iterator = familyMap.entrySet().iterator();
-        PColumnFamily[] families = new PColumnFamily[familyMap.size()];
-        ImmutableMap.Builder<String, PColumnFamily> familyByString = ImmutableMap.builder();
-        ImmutableSortedMap.Builder<byte[], PColumnFamily> familyByBytes = ImmutableSortedMap
-                .orderedBy(Bytes.BYTES_COMPARATOR);
-        for (int i = 0; i < families.length; i++) {
-            Map.Entry<PName,List<PColumn>> entry = iterator.next();
-            PColumnFamily family = new PColumnFamilyImpl(entry.getKey(), entry.getValue());
-            families[i] = family;
-            familyByString.put(family.getName().getString(), family);
-            familyByBytes.put(family.getName().getBytes(), family);
-            estimatedSize += family.getEstimatedSize();
-        }
-        this.families = ImmutableList.copyOf(families);
-        this.familyByBytes = familyByBytes.build();
-        this.familyByString = familyByString.build();
-        estimatedSize += SizedUtil.sizeOfArrayList(families.length);
-        estimatedSize += SizedUtil.sizeOfMap(families.length) * 2;
-        this.indexes = indexes == null ? Collections.<PTable>emptyList() : indexes;
-        for (PTable index : this.indexes) {
-            estimatedSize += index.getEstimatedSize();
-        }
-
-        this.parentSchemaName = parentSchemaName;
-        this.parentTableName = parentTableName;
-        this.parentName = parentTableName == null ? null : PNameFactory.newName(SchemaUtil.getTableName(
-            parentSchemaName!=null ? parentSchemaName.getString() : null, parentTableName.getString()));
-        estimatedSize += PNameFactory.getEstimatedSize(this.parentName);
-
-        this.physicalNames = physicalNames == null ? ImmutableList.<PName>of() : ImmutableList.copyOf(physicalNames);
-        for (PName name : this.physicalNames) {
-            estimatedSize += name.getEstimatedSize();
-        }
-        this.estimatedSize = estimatedSize;
-        this.baseColumnCount = baseColumnCount;
-        this.encodedCQCounter = encodedCQCounter;
-        this.useStatsForParallelization = useStatsForParallelization;
-        this.excludedColumns = excludedColumns == null ? ImmutableList.<PColumn>of() : ImmutableList.copyOf(excludedColumns);
+    public static void checkTenantId(PName tenantId) {
+        // tenantId should be null or not empty
+        Preconditions.checkArgument(tenantId == null || tenantId.getBytes().length > 0);
     }
 
     @Override
@@ -1340,9 +1555,9 @@ public class PTableImpl implements PTable {
         if (table.hasIsNamespaceMapped()) {
             isNamespaceMapped = table.getIsNamespaceMapped();
         }
-        String autoParititonSeqName = null;
+        String autoPartitionSeqName = null;
         if (table.hasAutoParititonSeqName()) {
-            autoParititonSeqName = table.getAutoParititonSeqName();
+            autoPartitionSeqName = table.getAutoParititonSeqName();
         }
         boolean isAppendOnlySchema = false;
         if (table.hasIsAppendOnlySchema()) {
@@ -1358,7 +1573,7 @@ public class PTableImpl implements PTable {
         if (table.hasEncodingScheme()) {
             qualifierEncodingScheme = QualifierEncodingScheme.fromSerializedValue(table.getEncodingScheme().toByteArray()[0]);
         }
-        EncodedCQCounter encodedColumnQualifierCounter = null;
+        EncodedCQCounter encodedColumnQualifierCounter;
         if ((!EncodedColumnsUtil.usesEncodedColumnNames(qualifierEncodingScheme) || tableType == PTableType.VIEW)) {
             encodedColumnQualifierCounter = PTable.EncodedCQCounter.NULL_COUNTER;
         }
@@ -1375,14 +1590,50 @@ public class PTableImpl implements PTable {
             useStatsForParallelization = table.getUseStatsForParallelization();
         }
         try {
-            PTableImpl result = new PTableImpl();
-            result.init(tenantId, schemaName, tableName, tableType, indexState, timeStamp, sequenceNumber, pkName,
-                (bucketNum == NO_SALTING) ? null : bucketNum, columns, parentSchemaName, parentTableName, indexes,
-                        isImmutableRows, physicalNames, defaultFamilyName, viewStatement, disableWAL,
-                        multiTenant, storeNulls, viewType, viewIndexType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable,
-                        transactionProvider, updateCacheFrequency, indexDisableTimestamp, isNamespaceMapped, autoParititonSeqName, 
-                        isAppendOnlySchema, storageScheme, qualifierEncodingScheme, encodedColumnQualifierCounter, useStatsForParallelization, null);
-            return result;
+            return new PTableImpl.Builder()
+                    .setType(tableType)
+                    .setState(indexState)
+                    .setTimeStamp(timeStamp)
+                    .setIndexDisableTimestamp(indexDisableTimestamp)
+                    .setSequenceNumber(sequenceNumber)
+                    .setImmutableRows(isImmutableRows)
+                    .setViewStatement(viewStatement)
+                    .setDisableWAL(disableWAL)
+                    .setMultiTenant(multiTenant)
+                    .setStoreNulls(storeNulls)
+                    .setViewType(viewType)
+                    .setViewIndexType(viewIndexType)
+                    .setViewIndexId(viewIndexId)
+                    .setIndexType(indexType)
+                    .setTransactionProvider(transactionProvider)
+                    .setUpdateCacheFrequency(updateCacheFrequency)
+                    .setNamespaceMapped(isNamespaceMapped)
+                    .setAutoPartitionSeqName(autoPartitionSeqName)
+                    .setAppendOnlySchema(isAppendOnlySchema)
+                    // null check for backward compatibility and sanity. If any of the two below is null,
+                    // then it means the table is a non-encoded table.
+                    .setImmutableStorageScheme(storageScheme == null ?
+                            ImmutableStorageScheme.ONE_CELL_PER_COLUMN : storageScheme)
+                    .setQualifierEncodingScheme(qualifierEncodingScheme == null ?
+                            QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : qualifierEncodingScheme)
+                    .setBaseColumnCount(baseColumnCount)
+                    .setEncodedCQCounter(encodedColumnQualifierCounter)
+                    .setUseStatsForParallelization(useStatsForParallelization)
+                    .setExcludedColumns(ImmutableList.<PColumn>of())
+                    .setTenantId(tenantId)
+                    .setSchemaName(schemaName)
+                    .setTableName(tableName)
+                    .setPkName(pkName)
+                    .setDefaultFamilyName(defaultFamilyName)
+                    .setRowKeyOrderOptimizable(rowKeyOrderOptimizable)
+                    .setBucketNum((bucketNum == NO_SALTING) ? null : bucketNum)
+                    .setIndexes(indexes == null ? Collections.<PTable>emptyList() : indexes)
+                    .setParentSchemaName(parentSchemaName)
+                    .setParentTableName(parentTableName)
+                    .setPhysicalNames(physicalNames == null ?
+                            ImmutableList.<PName>of() : ImmutableList.copyOf(physicalNames))
+                    .setColumns(columns)
+                    .build();
         } catch (SQLException e) {
             throw new RuntimeException(e); // Impossible
         }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
index 37cae22..b40c0b8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
@@ -17,6 +17,8 @@
  */
 package org.apache.phoenix.schema;
 
+import java.sql.SQLException;
+import java.util.Collections;
 import java.util.Objects;
 
 import org.apache.phoenix.compile.TupleProjectionCompiler;
@@ -28,7 +30,7 @@ import org.apache.phoenix.util.SchemaUtil;
 
 
 public class TableRef {
-    public static final TableRef EMPTY_TABLE_REF = new TableRef(new PTableImpl());
+    public static final TableRef EMPTY_TABLE_REF = createEmptyTableRef();
     
     private PTable table;
     private long upperBoundTimeStamp;
@@ -37,6 +39,19 @@ public class TableRef {
     private final boolean hasDynamicCols;
     private final long currentTime;
 
+    private static TableRef createEmptyTableRef() {
+        try {
+            return new TableRef(new PTableImpl.Builder()
+                    .setIndexes(Collections.<PTable>emptyList())
+                    .setPhysicalNames(Collections.<PName>emptyList())
+                    .setRowKeySchema(RowKeySchema.EMPTY_SCHEMA)
+                    .build());
+        } catch (SQLException e) {
+            // Should never happen
+            return null;
+        }
+    }
+
     public TableRef(TableRef tableRef) {
         this(tableRef.alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols);
     }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
index 26caa0d..34f82d2 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.phoenix.execute;
 
+import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
 import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY;
 import static org.apache.phoenix.util.PhoenixRuntime.CONNECTIONLESS;
 import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
@@ -30,6 +31,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Scan;
@@ -258,11 +260,31 @@ public class CorrelatePlanTest {
                     i, expr.getSortOrder(), null, null, false, name, false, false, colName.getBytes(), HConstants.LATEST_TIMESTAMP));
         }
         try {
-            PTable pTable = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME,
-                    PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM,
-                    null, null, columns, null, null, Collections.<PTable>emptyList(),
-                    false, Collections.<PName>emptyList(), null, null, false, false, false, null, null,
-                    null, null, true, null, 0, 0L, Boolean.FALSE, null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, EncodedCQCounter.NULL_COUNTER, true);
+            PTable pTable =  new PTableImpl.Builder()
+                    .setType(PTableType.SUBQUERY)
+                    .setTimeStamp(MetaDataProtocol.MIN_TABLE_TIMESTAMP)
+                    .setIndexDisableTimestamp(0L)
+                    .setSequenceNumber(PTable.INITIAL_SEQ_NUM)
+                    .setImmutableRows(false)
+                    .setDisableWAL(false)
+                    .setMultiTenant(false)
+                    .setStoreNulls(false)
+                    .setUpdateCacheFrequency(0)
+                    .setNamespaceMapped(Boolean.FALSE)
+                    .setAppendOnlySchema(false)
+                    .setImmutableStorageScheme(ImmutableStorageScheme.ONE_CELL_PER_COLUMN)
+                    .setQualifierEncodingScheme(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS)
+                    .setBaseColumnCount(BASE_TABLE_BASE_COLUMN_COUNT)
+                    .setEncodedCQCounter(EncodedCQCounter.NULL_COUNTER)
+                    .setUseStatsForParallelization(true)
+                    .setExcludedColumns(ImmutableList.<PColumn>of())
+                    .setSchemaName(PName.EMPTY_NAME)
+                    .setTableName(PName.EMPTY_NAME)
+                    .setRowKeyOrderOptimizable(true)
+                    .setIndexes(Collections.<PTable>emptyList())
+                    .setPhysicalNames(ImmutableList.<PName>of())
+                    .setColumns(columns)
+                    .build();
             TableRef sourceTable = new TableRef(pTable);
             List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList();
             for (PColumn column : sourceTable.getTable().getColumns()) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
index f400d0b..110bd26 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.phoenix.execute;
 
+import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
 import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY;
 import static org.apache.phoenix.util.PhoenixRuntime.CONNECTIONLESS;
 import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
@@ -30,6 +31,7 @@ import java.sql.SQLException;
 import java.util.Collections;
 import java.util.List;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Scan;
@@ -181,12 +183,33 @@ public class LiteralResultIteratorPlanTest {
                 HConstants.LATEST_TIMESTAMP));
         }
         try {
-            PTable pTable = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME, PTableType.SUBQUERY, null,
-                    MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null,
-                    Collections.<PTable> emptyList(), false, Collections.<PName> emptyList(), null, null, false, false,
-                    false, null, null, null, null, true, null, 0, 0L, false, null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, EncodedCQCounter.NULL_COUNTER, true);
+            PTable pTable = new PTableImpl.Builder()
+                    .setType(PTableType.SUBQUERY)
+                    .setTimeStamp(MetaDataProtocol.MIN_TABLE_TIMESTAMP)
+                    .setIndexDisableTimestamp(0L)
+                    .setSequenceNumber(PTable.INITIAL_SEQ_NUM)
+                    .setImmutableRows(false)
+                    .setDisableWAL(false)
+                    .setMultiTenant(false)
+                    .setStoreNulls(false)
+                    .setUpdateCacheFrequency(0)
+                    .setNamespaceMapped(false)
+                    .setAppendOnlySchema(false)
+                    .setImmutableStorageScheme(ImmutableStorageScheme.ONE_CELL_PER_COLUMN)
+                    .setQualifierEncodingScheme(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS)
+                    .setBaseColumnCount(BASE_TABLE_BASE_COLUMN_COUNT)
+                    .setEncodedCQCounter(EncodedCQCounter.NULL_COUNTER)
+                    .setUseStatsForParallelization(true)
+                    .setExcludedColumns(ImmutableList.<PColumn>of())
+                    .setSchemaName(PName.EMPTY_NAME)
+                    .setTableName(PName.EMPTY_NAME)
+                    .setRowKeyOrderOptimizable(true)
+                    .setIndexes(Collections.<PTable>emptyList())
+                    .setPhysicalNames(ImmutableList.<PName>of())
+                    .setColumns(columns)
+                    .build();
             TableRef sourceTable = new TableRef(pTable);
-            List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList();
+            List<ColumnRef> sourceColumnRefs = Lists.newArrayList();
             for (PColumn column : sourceTable.getTable().getColumns()) {
                 sourceColumnRefs.add(new ColumnRef(sourceTable, column.getPosition()));
             }


[20/22] phoenix git commit: PHOENIX-5013 Increase timeout for Tephra discovery service

Posted by ja...@apache.org.
PHOENIX-5013 Increase timeout for Tephra discovery service


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/169270da
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/169270da
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/169270da

Branch: refs/heads/omid2
Commit: 169270daa894628f3dacf2957ef6cfc8a99d2f6a
Parents: d1efd66
Author: James Taylor <ja...@apache.org>
Authored: Sat Nov 10 11:07:02 2018 -0800
Committer: James Taylor <ja...@apache.org>
Committed: Sat Nov 10 11:07:02 2018 -0800

----------------------------------------------------------------------
 .../apache/phoenix/query/QueryServicesTestImpl.java   |  6 +++---
 .../test/java/org/apache/phoenix/util/TestUtil.java   | 14 ++++++++++++++
 2 files changed, 17 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/169270da/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
index 841abb6..49fb8e8 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
@@ -25,8 +25,8 @@ import org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec;
 import org.apache.phoenix.transaction.OmidTransactionProvider;
 import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.TestUtil;
 import org.apache.tephra.TxConstants;
-import org.apache.twill.internal.utils.Networks;
 
 
 /**
@@ -129,12 +129,12 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
                 .set(TxConstants.Manager.CFG_DO_PERSIST, false)
                 .set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times")
                 .set(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1)
-                .set(TxConstants.Service.CFG_DATA_TX_BIND_PORT, Networks.getRandomPort())
+                .set(TxConstants.Service.CFG_DATA_TX_BIND_PORT, TestUtil.getRandomPort())
                 .set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, Files.createTempDir().getAbsolutePath())
                 .set(TxConstants.Manager.CFG_TX_TIMEOUT, DEFAULT_TXN_TIMEOUT_SECONDS)
                 .set(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, 5L)
                 // setup default test configs for Omid
-                .set(OmidTransactionProvider.OMID_TSO_PORT, Networks.getRandomPort())
+                .set(OmidTransactionProvider.OMID_TSO_PORT, TestUtil.getRandomPort())
                 ;
     }
     

http://git-wip-us.apache.org/repos/asf/phoenix/blob/169270da/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index f0a26b9..f3faa0c 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -36,6 +36,7 @@ import static org.junit.Assert.fail;
 import java.io.File;
 import java.io.IOException;
 import java.math.BigDecimal;
+import java.net.ServerSocket;
 import java.sql.Connection;
 import java.sql.Date;
 import java.sql.DriverManager;
@@ -1105,4 +1106,17 @@ public class TestUtil {
         }
         return filteredData;
     }
+    
+    /**
+     * Find a random free port in localhost for binding.
+     * @return A port number or -1 for failure.
+     */
+    public static int getRandomPort() {
+        try (ServerSocket socket = new ServerSocket(0)) {
+            socket.setReuseAddress(true);
+            return socket.getLocalPort();
+        } catch (IOException e) {
+            return -1;
+        }
+    }
 }


[18/22] phoenix git commit: Revert "PHOENIX-4971 Drop index will execute successfully using Incorrect name of parent tables"

Posted by ja...@apache.org.
Revert "PHOENIX-4971 Drop index will execute successfully using Incorrect name of parent tables"

This reverts commit 527098ed621d62ba18fc19099ec67d8b29e027f4.


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/fd7ae940
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/fd7ae940
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/fd7ae940

Branch: refs/heads/omid2
Commit: fd7ae94065ae33f1bb20bca60b7756595f8b10b9
Parents: d6083ae
Author: Thomas D'Silva <td...@apache.org>
Authored: Wed Nov 7 11:10:05 2018 -0800
Committer: Thomas D'Silva <td...@apache.org>
Committed: Wed Nov 7 11:10:05 2018 -0800

----------------------------------------------------------------------
 .../phoenix/end2end/index/DropMetadataIT.java   | 23 +-------------------
 .../phoenix/exception/SQLExceptionCode.java     |  2 --
 .../apache/phoenix/schema/MetaDataClient.java   | 15 -------------
 3 files changed, 1 insertion(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/fd7ae940/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
index 3c670c5..b92ed8d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
@@ -18,12 +18,9 @@
 package org.apache.phoenix.end2end.index;
 
 import static org.apache.phoenix.util.TestUtil.HBASE_NATIVE_SCHEMA_NAME;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 import java.sql.Connection;
 import java.sql.DriverManager;
-import java.sql.SQLException;
 import java.util.Properties;
 
 import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -32,7 +29,6 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
-import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.util.PropertiesUtil;
@@ -60,24 +56,7 @@ public class DropMetadataIT extends ParallelStatsDisabledIT {
         String url = QueryUtil.getConnectionUrl(props, config, PRINCIPAL);
         return DriverManager.getConnection(url, props);
     }
-
-    @Test
-    public void testDropIndexTableHasSameNameWithDataTable() {
-        String tableName = generateUniqueName();
-        String indexName = "IDX_" + tableName;
-        try (Connection conn = DriverManager.getConnection(getUrl())) {
-            String createTable = "CREATE TABLE " + tableName + "  (id varchar not null primary key, col integer)";
-            conn.createStatement().execute(createTable);
-            String createIndex = "CREATE INDEX " + indexName + " on " + tableName + "(col)";
-            conn.createStatement().execute(createIndex);
-            String dropIndex = "DROP INDEX " + indexName + " on " + indexName;
-            conn.createStatement().execute(dropIndex);
-            fail("should not execute successfully");
-        } catch (SQLException e) {
-            assertTrue(SQLExceptionCode.PARENT_TABLE_NOT_FOUND.getErrorCode() == e.getErrorCode());
-        }
-    }
-
+    
     @Test
     public void testDropViewKeepsHTable() throws Exception {
         Connection conn = getConnection();

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fd7ae940/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index 5bffed5..d557714 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -185,8 +185,6 @@ public enum SQLExceptionCode {
      INVALID_REPLAY_AT(533, "42910", "Value of REPLAY_AT cannot be less than zero."),
      UNEQUAL_SCN_AND_BUILD_INDEX_AT(534, "42911", "If both specified, values of CURRENT_SCN and BUILD_INDEX_AT must be equal."),
      ONLY_INDEX_UPDATABLE_AT_SCN(535, "42912", "Only an index may be updated when the BUILD_INDEX_AT property is specified"),
-     PARENT_TABLE_NOT_FOUND(536, "42913", "Can't drop the index because the parent table in the DROP statement is incorrect."),
-
      /**
      * HBase and Phoenix specific implementation defined sub-classes.
      * Column family related exceptions.

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fd7ae940/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index aae9fc4..3b68247 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -23,7 +23,6 @@ import static org.apache.hadoop.hbase.HColumnDescriptor.TTL;
 import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.ANALYZE_TABLE;
 import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC_ATTRIB;
 import static org.apache.phoenix.exception.SQLExceptionCode.INSUFFICIENT_MULTI_TENANT_COLUMNS;
-import static org.apache.phoenix.exception.SQLExceptionCode.PARENT_TABLE_NOT_FOUND;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE;
@@ -3125,20 +3124,6 @@ public class MetaDataClient {
     }
     MutationState dropTable(String schemaName, String tableName, String parentTableName, PTableType tableType,
             boolean ifExists, boolean cascade, boolean skipAddingParentColumns) throws SQLException {
-        // Checking the parent table whether exists
-        String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
-        try {
-            PTable ptable = connection.getTable(new PTableKey(connection.getTenantId(), fullTableName));
-            if (parentTableName != null &&!parentTableName.equals(ptable.getParentTableName().getString())) {
-                throw new SQLExceptionInfo.Builder(PARENT_TABLE_NOT_FOUND)
-                        .setSchemaName(schemaName).setTableName(tableName).build().buildException();
-            }
-        } catch (TableNotFoundException e) {
-            if (!ifExists) {
-                throw e;
-            }
-        }
-
         connection.rollback();
         boolean wasAutoCommit = connection.getAutoCommit();
         PName tenantId = connection.getTenantId();


[03/22] phoenix git commit: PHOENIX-4971 Drop index will execute successfully using Incorrect name of parent tables

Posted by ja...@apache.org.
PHOENIX-4971 Drop index will execute successfully using Incorrect name of parent tables

Signed-off-by: Geoffrey Jacoby <gj...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/527098ed
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/527098ed
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/527098ed

Branch: refs/heads/omid2
Commit: 527098ed621d62ba18fc19099ec67d8b29e027f4
Parents: 3903ad7
Author: Jaanai <cl...@gmail.com>
Authored: Sun Oct 28 19:10:51 2018 +0800
Committer: Geoffrey Jacoby <gj...@apache.org>
Committed: Sun Oct 28 15:28:39 2018 -0700

----------------------------------------------------------------------
 .../phoenix/end2end/index/DropMetadataIT.java   | 23 +++++++++++++++++++-
 .../phoenix/exception/SQLExceptionCode.java     |  2 ++
 .../apache/phoenix/schema/MetaDataClient.java   | 15 +++++++++++++
 3 files changed, 39 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/527098ed/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
index b92ed8d..3c670c5 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
@@ -18,9 +18,12 @@
 package org.apache.phoenix.end2end.index;
 
 import static org.apache.phoenix.util.TestUtil.HBASE_NATIVE_SCHEMA_NAME;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import java.sql.Connection;
 import java.sql.DriverManager;
+import java.sql.SQLException;
 import java.util.Properties;
 
 import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -29,6 +32,7 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
+import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.util.PropertiesUtil;
@@ -56,7 +60,24 @@ public class DropMetadataIT extends ParallelStatsDisabledIT {
         String url = QueryUtil.getConnectionUrl(props, config, PRINCIPAL);
         return DriverManager.getConnection(url, props);
     }
-    
+
+    @Test
+    public void testDropIndexTableHasSameNameWithDataTable() {
+        String tableName = generateUniqueName();
+        String indexName = "IDX_" + tableName;
+        try (Connection conn = DriverManager.getConnection(getUrl())) {
+            String createTable = "CREATE TABLE " + tableName + "  (id varchar not null primary key, col integer)";
+            conn.createStatement().execute(createTable);
+            String createIndex = "CREATE INDEX " + indexName + " on " + tableName + "(col)";
+            conn.createStatement().execute(createIndex);
+            String dropIndex = "DROP INDEX " + indexName + " on " + indexName;
+            conn.createStatement().execute(dropIndex);
+            fail("should not execute successfully");
+        } catch (SQLException e) {
+            assertTrue(SQLExceptionCode.PARENT_TABLE_NOT_FOUND.getErrorCode() == e.getErrorCode());
+        }
+    }
+
     @Test
     public void testDropViewKeepsHTable() throws Exception {
         Connection conn = getConnection();

http://git-wip-us.apache.org/repos/asf/phoenix/blob/527098ed/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index d84857d..6696521 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -184,6 +184,8 @@ public enum SQLExceptionCode {
      INVALID_REPLAY_AT(533, "42910", "Value of REPLAY_AT cannot be less than zero."),
      UNEQUAL_SCN_AND_BUILD_INDEX_AT(534, "42911", "If both specified, values of CURRENT_SCN and BUILD_INDEX_AT must be equal."),
      ONLY_INDEX_UPDATABLE_AT_SCN(535, "42912", "Only an index may be updated when the BUILD_INDEX_AT property is specified"),
+     PARENT_TABLE_NOT_FOUND(536, "42913", "Can't drop the index because the parent table in the DROP statement is incorrect."),
+
      /**
      * HBase and Phoenix specific implementation defined sub-classes.
      * Column family related exceptions.

http://git-wip-us.apache.org/repos/asf/phoenix/blob/527098ed/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 66e8baa..6563c40 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -23,6 +23,7 @@ import static org.apache.hadoop.hbase.HColumnDescriptor.TTL;
 import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.ANALYZE_TABLE;
 import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC_ATTRIB;
 import static org.apache.phoenix.exception.SQLExceptionCode.INSUFFICIENT_MULTI_TENANT_COLUMNS;
+import static org.apache.phoenix.exception.SQLExceptionCode.PARENT_TABLE_NOT_FOUND;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE;
@@ -3036,6 +3037,20 @@ public class MetaDataClient {
     }
     MutationState dropTable(String schemaName, String tableName, String parentTableName, PTableType tableType,
             boolean ifExists, boolean cascade, boolean skipAddingParentColumns) throws SQLException {
+        // Checking the parent table whether exists
+        String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
+        try {
+            PTable ptable = connection.getTable(new PTableKey(connection.getTenantId(), fullTableName));
+            if (parentTableName != null &&!parentTableName.equals(ptable.getParentTableName().getString())) {
+                throw new SQLExceptionInfo.Builder(PARENT_TABLE_NOT_FOUND)
+                        .setSchemaName(schemaName).setTableName(tableName).build().buildException();
+            }
+        } catch (TableNotFoundException e) {
+            if (!ifExists) {
+                throw e;
+            }
+        }
+
         connection.rollback();
         boolean wasAutoCommit = connection.getAutoCommit();
         PName tenantId = connection.getTenantId();


[08/22] phoenix git commit: PHOENIX-4989 Include disruptor jar in shaded dependency

Posted by ja...@apache.org.
PHOENIX-4989 Include disruptor jar in shaded dependency


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c3ed4f84
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c3ed4f84
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c3ed4f84

Branch: refs/heads/omid2
Commit: c3ed4f84444f80ea9825f1b6f1db0933ceddcd22
Parents: fc35cce
Author: Aman Poonia <am...@gmail.com>
Authored: Tue Oct 30 13:57:52 2018 -0700
Committer: Vincent Poon <vi...@apache.org>
Committed: Thu Nov 1 17:21:15 2018 -0700

----------------------------------------------------------------------
 phoenix-server/pom.xml | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/c3ed4f84/phoenix-server/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-server/pom.xml b/phoenix-server/pom.xml
index f5ba7f7..339bf8a 100644
--- a/phoenix-server/pom.xml
+++ b/phoenix-server/pom.xml
@@ -139,6 +139,7 @@
                   <include>com.ibm.icu:icu4j</include>
                   <include>com.ibm.icu:icu4j-charset</include>
                   <include>com.ibm.icu:icu4j-localespi</include>
+                  <include>com.lmax:disruptor</include>
                 </includes>
                   <excludes>
                     <exclude>org.apache.phoenix:phoenix-server</exclude>


[11/22] phoenix git commit: PHOENIX-4981 Add tests for ORDER BY, GROUP BY and salted tables using phoenix-spark

Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableIT.java
index c9168f1..69c9869 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableIT.java
@@ -37,104 +37,18 @@ import org.apache.phoenix.util.QueryUtil;
 import org.apache.phoenix.util.SchemaUtil;
 import org.junit.Test;
 
-
 /**
  * Tests for table with transparent salting.
  */
 
-public class SaltedTableIT extends ParallelStatsDisabledIT {
-
-	private static String getUniqueTableName() {
-		return SchemaUtil.getTableName(generateUniqueName(), generateUniqueName());
-	}
-	
-    private static String initTableValues(byte[][] splits) throws Exception {
-    	String tableName = getUniqueTableName();    	
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        
-        // Rows we inserted:
-        // 1ab123abc111
-        // 1abc456abc111
-        // 1de123abc111
-        // 2abc123def222 
-        // 3abc123ghi333
-        // 4abc123jkl444
-        try {
-            // Upsert with no column specifies.
-            ensureTableCreated(getUrl(), tableName, TABLE_WITH_SALTING, splits, null, null);
-            String query = "UPSERT INTO " + tableName + " VALUES(?,?,?,?,?)";
-            PreparedStatement stmt = conn.prepareStatement(query);
-            stmt.setInt(1, 1);
-            stmt.setString(2, "ab");
-            stmt.setString(3, "123");
-            stmt.setString(4, "abc");
-            stmt.setInt(5, 111);
-            stmt.execute();
-            conn.commit();
-            
-            stmt.setInt(1, 1);
-            stmt.setString(2, "abc");
-            stmt.setString(3, "456");
-            stmt.setString(4, "abc");
-            stmt.setInt(5, 111);
-            stmt.execute();
-            conn.commit();
-            
-            // Test upsert when statement explicitly specifies the columns to upsert into.
-            query = "UPSERT INTO " + tableName +
-                    " (a_integer, a_string, a_id, b_string, b_integer) " + 
-                    " VALUES(?,?,?,?,?)";
-            stmt = conn.prepareStatement(query);
-            
-            stmt.setInt(1, 1);
-            stmt.setString(2, "de");
-            stmt.setString(3, "123");
-            stmt.setString(4, "abc");
-            stmt.setInt(5, 111);
-            stmt.execute();
-            conn.commit();
-            
-            stmt.setInt(1, 2);
-            stmt.setString(2, "abc");
-            stmt.setString(3, "123");
-            stmt.setString(4, "def");
-            stmt.setInt(5, 222);
-            stmt.execute();
-            conn.commit();
-            
-            // Test upsert when order of column is shuffled.
-            query = "UPSERT INTO " + tableName +
-                    " (a_string, a_integer, a_id, b_string, b_integer) " + 
-                    " VALUES(?,?,?,?,?)";
-            stmt = conn.prepareStatement(query);
-            stmt.setString(1, "abc");
-            stmt.setInt(2, 3);
-            stmt.setString(3, "123");
-            stmt.setString(4, "ghi");
-            stmt.setInt(5, 333);
-            stmt.execute();
-            conn.commit();
-            
-            stmt.setString(1, "abc");
-            stmt.setInt(2, 4);
-            stmt.setString(3, "123");
-            stmt.setString(4, "jkl");
-            stmt.setInt(5, 444);
-            stmt.execute();
-            conn.commit();
-        } finally {
-            conn.close();
-        }
-        return tableName;
-    }
+public class SaltedTableIT extends BaseSaltedTableIT {
 
     @Test
     public void testTableWithInvalidBucketNumber() throws Exception {
         Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
         Connection conn = DriverManager.getConnection(getUrl(), props);
         try {
-            String query = "create table " + getUniqueTableName() + " (a_integer integer not null CONSTRAINT pk PRIMARY KEY (a_integer)) SALT_BUCKETS = 257";
+            String query = "create table " + generateUniqueName() + " (a_integer integer not null CONSTRAINT pk PRIMARY KEY (a_integer)) SALT_BUCKETS = 257";
             PreparedStatement stmt = conn.prepareStatement(query);
             stmt.execute();
             fail("Should have caught exception");
@@ -148,370 +62,12 @@ public class SaltedTableIT extends ParallelStatsDisabledIT {
     @Test
     public void testTableWithSplit() throws Exception {
         try {
-            createTestTable(getUrl(), "create table " + getUniqueTableName() + " (a_integer integer not null primary key) SALT_BUCKETS = 4",
+            createTestTable(getUrl(), "create table " + generateUniqueName() + " (a_integer integer not null primary key) SALT_BUCKETS = 4",
                     new byte[][] {{1}, {2,3}, {2,5}, {3}}, null);
             fail("Should have caught exception");
         } catch (SQLException e) {
             assertTrue(e.getMessage(), e.getMessage().contains("ERROR 1022 (42Y81): Should not specify split points on salted table with default row key order."));
         }
     }
-    
-    @Test
-    public void testSelectValueNoWhereClause() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        try {
-            String tableName = initTableValues(null);
-            
-            String query = "SELECT * FROM " + tableName;
-            PreparedStatement statement = conn.prepareStatement(query);
-            ResultSet rs = statement.executeQuery();
-            
-            assertTrue(rs.next());
-            assertEquals(1, rs.getInt(1));
-            assertEquals("ab", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("abc", rs.getString(4));
-            assertEquals(111, rs.getInt(5));
-            
-            assertTrue(rs.next());
-            assertEquals(1, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            assertEquals("456", rs.getString(3));
-            assertEquals("abc", rs.getString(4));
-            assertEquals(111, rs.getInt(5));
-            
-            assertTrue(rs.next());
-            assertEquals(1, rs.getInt(1));
-            assertEquals("de", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("abc", rs.getString(4));
-            assertEquals(111, rs.getInt(5));
-            
-            assertTrue(rs.next());
-            assertEquals(2, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("def", rs.getString(4));
-            assertEquals(222, rs.getInt(5));
-            
-            assertTrue(rs.next());
-            assertEquals(3, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("ghi", rs.getString(4));
-            assertEquals(333, rs.getInt(5));
-            
-            assertTrue(rs.next());
-            assertEquals(4, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("jkl", rs.getString(4));
-            assertEquals(444, rs.getInt(5));
-            
-            assertFalse(rs.next());
-        } finally {
-            conn.close();
-        }
-    }
-
-    @Test
-    public void testSelectValueWithFullyQualifiedWhereClause() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        try {
-            String tableName = initTableValues(null);
-            String query;
-            PreparedStatement stmt;
-            ResultSet rs;
-            
-            // Variable length slot with bounded ranges.
-            query = "SELECT * FROM " + tableName + 
-                    " WHERE a_integer = 1 AND a_string >= 'ab' AND a_string < 'de' AND a_id = '123'";
-            stmt = conn.prepareStatement(query);
-            rs = stmt.executeQuery();
-            assertTrue(rs.next());
-            assertEquals(1, rs.getInt(1));
-            assertEquals("ab", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("abc", rs.getString(4));
-            assertEquals(111, rs.getInt(5));
-            assertFalse(rs.next());
-
-            // all single slots with one value.
-            query = "SELECT * FROM " + tableName + 
-                    " WHERE a_integer = 1 AND a_string = 'ab' AND a_id = '123'";
-            stmt = conn.prepareStatement(query);
-            
-            rs = stmt.executeQuery();
-            assertTrue(rs.next());
-            assertEquals(1, rs.getInt(1));
-            assertEquals("ab", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("abc", rs.getString(4));
-            assertEquals(111, rs.getInt(5));
-            assertFalse(rs.next());
-            
-            // all single slots with multiple values.
-            query = "SELECT * FROM " + tableName + 
-                    " WHERE a_integer in (2, 4) AND a_string = 'abc' AND a_id = '123'";
-            stmt = conn.prepareStatement(query);
-            rs = stmt.executeQuery();
-            
-            assertTrue(rs.next());
-            assertEquals(2, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("def", rs.getString(4));
-            assertEquals(222, rs.getInt(5));
-            
-            assertTrue(rs.next());
-            assertEquals(4, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("jkl", rs.getString(4));
-            assertEquals(444, rs.getInt(5));
-            assertFalse(rs.next());
-            
-            query = "SELECT a_integer, a_string FROM " + tableName +
-                    " WHERE a_integer in (1,2,3,4) AND a_string in ('a', 'abc', 'de') AND a_id = '123'";
-            stmt = conn.prepareStatement(query);
-            rs = stmt.executeQuery();
-            
-            assertTrue(rs.next());
-            assertEquals(1, rs.getInt(1));
-            assertEquals("de", rs.getString(2));
-            
-            assertTrue(rs.next());
-            assertEquals(2, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            
-            assertTrue(rs.next());
-            assertEquals(3, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            
-            assertTrue(rs.next());
-            assertEquals(4, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            assertFalse(rs.next());
-            
-            // fixed length slot with bounded ranges.
-            query = "SELECT a_string, a_id FROM " + tableName + 
-                    " WHERE a_integer > 1 AND a_integer < 4 AND a_string = 'abc' AND a_id = '123'";
-            stmt = conn.prepareStatement(query);
-            rs = stmt.executeQuery();
-            assertTrue(rs.next());
-            assertEquals("abc", rs.getString(1));
-            assertEquals("123", rs.getString(2));
-            
-            assertTrue(rs.next());
-            assertEquals("abc", rs.getString(1));
-            assertEquals("123", rs.getString(2));
-            assertFalse(rs.next());
-            
-            // fixed length slot with unbound ranges.
-            query = "SELECT b_string, b_integer FROM " + tableName + 
-                    " WHERE a_integer > 1 AND a_string = 'abc' AND a_id = '123'";
-            stmt = conn.prepareStatement(query);
-            rs = stmt.executeQuery();
-            assertTrue(rs.next());
-            assertEquals("def", rs.getString(1));
-            assertEquals(222, rs.getInt(2));
-            
-            assertTrue(rs.next());
-            assertEquals("ghi", rs.getString(1));
-            assertEquals(333, rs.getInt(2));
-            
-            assertTrue(rs.next());
-            assertEquals("jkl", rs.getString(1));
-            assertEquals(444, rs.getInt(2));
-            assertFalse(rs.next());
-            
-            // Variable length slot with unbounded ranges.
-            query = "SELECT * FROM " + tableName + 
-                    " WHERE a_integer = 1 AND a_string > 'ab' AND a_id = '123'";
-            stmt = conn.prepareStatement(query);
-            rs = stmt.executeQuery();
-            assertTrue(rs.next());
-            assertEquals(1, rs.getInt(1));
-            assertEquals("de", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("abc", rs.getString(4));
-            assertEquals(111, rs.getInt(5));
-            assertFalse(rs.next());
-
-        } finally {
-            conn.close();
-        }
-    }
-
-    @Test
-    public void testSelectValueWithNotFullyQualifiedWhereClause() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        try {
-            String tableName = initTableValues(null);
-            
-            // Where without fully qualified key, point query.
-            String query = "SELECT * FROM " + tableName + " WHERE a_integer = ? AND a_string = ?";
-            PreparedStatement stmt = conn.prepareStatement(query);
-            
-            stmt.setInt(1, 1);
-            stmt.setString(2, "abc");
-            ResultSet rs = stmt.executeQuery();
-            assertTrue(rs.next());
-            assertEquals(1, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            assertEquals("456", rs.getString(3));
-            assertEquals("abc", rs.getString(4));
-            assertEquals(111, rs.getInt(5));
-            assertFalse(rs.next());
-            
-            // Where without fully qualified key, range query.
-            query = "SELECT * FROM " + tableName + " WHERE a_integer >= 2";
-            stmt = conn.prepareStatement(query);
-            rs = stmt.executeQuery();
-            assertTrue(rs.next());
-            assertEquals(2, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("def", rs.getString(4));
-            assertEquals(222, rs.getInt(5));
-            
-            assertTrue(rs.next());
-            assertEquals(3, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("ghi", rs.getString(4));
-            assertEquals(333, rs.getInt(5));
-            
-            assertTrue(rs.next());
-            assertEquals(4, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("jkl", rs.getString(4));
-            assertEquals(444, rs.getInt(5));
-            assertFalse(rs.next());
-            
-            // With point query.
-            query = "SELECT a_string FROM " + tableName + " WHERE a_string = ?";
-            stmt = conn.prepareStatement(query);
-            stmt.setString(1, "de");
-            rs = stmt.executeQuery();
-            assertTrue(rs.next());
-            assertEquals("de", rs.getString(1));
-            assertFalse(rs.next());
-            
-            query = "SELECT a_id FROM " + tableName + " WHERE a_id = ?";
-            stmt = conn.prepareStatement(query);
-            stmt.setString(1, "456");
-            rs = stmt.executeQuery();
-            assertTrue(rs.next());
-            assertEquals("456", rs.getString(1));
-            assertFalse(rs.next());
-        } finally {
-            conn.close();
-        }
-    }
-
-    @Test
-    public void testSelectWithGroupBy() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        try {
-            String tableName = initTableValues(null);
-            
-            String query = "SELECT a_integer FROM " + tableName + " GROUP BY a_integer";
-            PreparedStatement stmt = conn.prepareStatement(query);
-            ResultSet rs = stmt.executeQuery();
-            int count = 0;
-            while (rs.next()) {
-                count++;
-            }
-            assertEquals("Group by does not return the right count.", count, 4);
-        } finally {
-            conn.close();
-        }
-    }
 
-    @Test
-    public void testLimitScan() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        try {
-            String tableName = initTableValues(null);
-            
-            String query = "SELECT a_integer FROM " + tableName + " WHERE a_string='abc' LIMIT 1";
-            PreparedStatement stmt = conn.prepareStatement(query);
-            ResultSet rs = stmt.executeQuery();
-            assertTrue(rs.next());
-            assertEquals(1, rs.getInt(1));
-            assertFalse(rs.next());
-        } finally {
-            conn.close();
-        }
-    }
-    
-    @Test
-    public void testSelectWithOrderByRowKey() throws Exception {
-        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-        Connection conn = DriverManager.getConnection(getUrl(), props);
-        try {
-            String tableName = initTableValues(null);
-            
-            String query = "SELECT * FROM " + tableName + " ORDER  BY  a_integer, a_string, a_id";
-            PreparedStatement statement = conn.prepareStatement(query);
-            ResultSet explainPlan = statement.executeQuery("EXPLAIN " + query);
-            // Confirm that ORDER BY in row key order will be optimized out for salted table
-            assertEquals("CLIENT PARALLEL 4-WAY FULL SCAN OVER " + tableName + "\n" + 
-                    "CLIENT MERGE SORT", QueryUtil.getExplainPlan(explainPlan));
-            ResultSet rs = statement.executeQuery();
-            
-            assertTrue(rs.next());
-            assertEquals(1, rs.getInt(1));
-            assertEquals("ab", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("abc", rs.getString(4));
-            assertEquals(111, rs.getInt(5));
-            
-            assertTrue(rs.next());
-            assertEquals(1, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            assertEquals("456", rs.getString(3));
-            assertEquals("abc", rs.getString(4));
-            assertEquals(111, rs.getInt(5));
-            
-            assertTrue(rs.next());
-            assertEquals(1, rs.getInt(1));
-            assertEquals("de", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("abc", rs.getString(4));
-            assertEquals(111, rs.getInt(5));
-            
-            assertTrue(rs.next());
-            assertEquals(2, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("def", rs.getString(4));
-            assertEquals(222, rs.getInt(5));
-            
-            assertTrue(rs.next());
-            assertEquals(3, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("ghi", rs.getString(4));
-            assertEquals(333, rs.getInt(5));
-            
-            assertTrue(rs.next());
-            assertEquals(4, rs.getInt(1));
-            assertEquals("abc", rs.getString(2));
-            assertEquals("123", rs.getString(3));
-            assertEquals("jkl", rs.getString(4));
-            assertEquals(444, rs.getInt(5));
-            
-            assertFalse(rs.next());
-        } finally {
-            conn.close();
-        }
-    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/main/java/org/apache/phoenix/util/QueryBuilder.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryBuilder.java
new file mode 100644
index 0000000..afce0dd
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryBuilder.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.phoenix.parse.HintNode;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.phoenix.util.SchemaUtil.getEscapedFullColumnName;
+
+public class QueryBuilder {
+
+    private String fullTableName;
+    // regular columns that are in the select clause
+    private List<String> selectColumns = Collections.emptyList();
+
+    // columns that are required for expressions in the select clause
+    private List<String> selectExpressionColumns  = Collections.emptyList();
+    // expression string in the select clause (for eg COL1 || COL2)
+    private String selectExpression;
+    private String whereClause;
+    private String orderByClause;
+    private String groupByClause;
+    private String havingClause;
+    private HintNode.Hint hint;
+    private boolean escapeCols;
+    private boolean distinct;
+    private int limit;
+
+    public String getFullTableName() {
+        return fullTableName;
+    }
+
+    /**
+     * @return column names required to evaluate this select statement
+     */
+    public List<String> getRequiredColumns() {
+        List<String> allColumns = Lists.newArrayList(selectColumns);
+        if (!CollectionUtils.isEmpty(selectExpressionColumns)) {
+            allColumns.addAll(selectExpressionColumns);
+        }
+        return allColumns;
+    }
+
+    public String getWhereClause() {
+        return whereClause;
+    }
+
+    public HintNode.Hint getHint() {
+        return hint;
+    }
+
+    public String getOrderByClause() {
+        return orderByClause;
+    }
+
+    public String getGroupByClause() {
+        return groupByClause;
+    }
+
+    public QueryBuilder setOrderByClause(String orderByClause) {
+        this.orderByClause = orderByClause;
+        return this;
+    }
+
+    public QueryBuilder setFullTableName(String fullTableName) {
+        this.fullTableName = fullTableName;
+        return this;
+    }
+
+    public QueryBuilder setSelectColumns(List<String> columns) {
+        this.selectColumns = columns;
+        return this;
+    }
+
+    public QueryBuilder setWhereClause(String whereClause) {
+        this.whereClause = whereClause;
+        return this;
+    }
+
+    public QueryBuilder setHint(HintNode.Hint hint) {
+        this.hint = hint;
+        return this;
+    }
+
+    public QueryBuilder setEscapeCols(boolean escapeCols) {
+        this.escapeCols = escapeCols;
+        return this;
+    }
+
+    public QueryBuilder setGroupByClause(String groupByClause) {
+        this.groupByClause = groupByClause;
+        return this;
+    }
+
+    public QueryBuilder setHavingClause(String havingClause) {
+        this.havingClause = havingClause;
+        return this;
+    }
+
+    public List<String> getSelectExpressionColumns() {
+        return selectExpressionColumns;
+    }
+
+    public QueryBuilder setSelectExpressionColumns(List<String> selectExpressionColumns) {
+        this.selectExpressionColumns = selectExpressionColumns;
+        return this;
+    }
+
+    public String getSelectExpression() {
+        return selectExpression;
+    }
+
+    public QueryBuilder setSelectExpression(String selectExpression) {
+        this.selectExpression = selectExpression;
+        return this;
+    }
+
+    public QueryBuilder setDistinct(boolean distinct) {
+        this.distinct = distinct;
+        return this;
+    }
+
+    public QueryBuilder setLimit(int limit) {
+        this.limit = limit;
+        return this;
+    }
+
+    public String build() {
+        Preconditions.checkNotNull(fullTableName, "Table name cannot be null");
+        if (CollectionUtils.isEmpty(selectColumns) && StringUtils.isBlank(selectExpression)) {
+            throw new IllegalArgumentException("At least one column or select expression must be provided");
+        }
+        StringBuilder query = new StringBuilder();
+        query.append("SELECT ");
+
+        if (distinct) {
+            query.append(" DISTINCT ");
+        }
+
+        if (hint != null) {
+            final HintNode node = new HintNode(hint.name());
+            String hintStr = node.toString();
+            query.append(hintStr);
+        }
+
+        StringBuilder selectClauseBuilder = new StringBuilder();
+        if (StringUtils.isNotBlank(selectExpression)) {
+            if (selectClauseBuilder.length()!=0) {
+                selectClauseBuilder.append(" , ");
+            }
+            selectClauseBuilder.append(selectExpression);
+        }
+
+        boolean first = true;
+        for (String col : selectColumns) {
+            if (StringUtils.isNotBlank(col)) {
+                if ((first && selectClauseBuilder.length()!=0) || !first) {
+                    selectClauseBuilder.append(" , ");
+                }
+                String fullColumnName = col;
+                if (escapeCols) {
+                    fullColumnName = getEscapedFullColumnName(col);
+                }
+                selectClauseBuilder.append(fullColumnName);
+                first = false;
+            }
+        }
+
+        query.append(selectClauseBuilder);
+        query.append(" FROM ");
+        query.append(fullTableName);
+        if (StringUtils.isNotBlank(whereClause)) {
+            query.append(" WHERE (").append(whereClause).append(")");
+        }
+        if (StringUtils.isNotBlank(groupByClause)) {
+            query.append(" GROUP BY ").append(groupByClause);
+        }
+        if (StringUtils.isNotBlank(havingClause)) {
+            query.append(" HAVING ").append(havingClause);
+        }
+        if (StringUtils.isNotBlank(orderByClause)) {
+            query.append(" ORDER BY ").append(orderByClause);
+        }
+        if (limit > 0) {
+            query.append(" LIMIT ").append(limit);
+        }
+        return query.toString();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java
index 94cbfea..4501158 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java
@@ -217,45 +217,15 @@ public final class QueryUtil {
      *
      * @param fullTableName name of the table for which the select statement needs to be created.
      * @param columns list of columns to be projected in the select statement.
-     * @param conditions The condition clause to be added to the WHERE condition
+     * @param whereClause The condition clause to be added to the WHERE condition
      * @param hint hint to use
      * @param escapeCols whether to escape the projected columns
      * @return Select Query
      */
     public static String constructSelectStatement(String fullTableName, List<String> columns,
-            final String conditions, Hint hint, boolean escapeCols) {
-        Preconditions.checkNotNull(fullTableName, "Table name cannot be null");
-        if (columns == null || columns.isEmpty()) {
-            throw new IllegalArgumentException("At least one column must be provided");
-        }
-        StringBuilder query = new StringBuilder();
-        query.append("SELECT ");
-
-        String hintStr = "";
-        if (hint != null) {
-            final HintNode node = new HintNode(hint.name());
-            hintStr = node.toString();
-        }
-        query.append(hintStr);
-
-        for (String col : columns) {
-            if (col != null) {
-                String fullColumnName = col;
-                if (escapeCols) {
-                    fullColumnName = getEscapedFullColumnName(col);
-                }
-                query.append(fullColumnName);
-                query.append(",");
-            }
-        }
-        // Remove the trailing comma
-        query.setLength(query.length() - 1);
-        query.append(" FROM ");
-        query.append(fullTableName);
-        if (conditions != null && conditions.length() > 0) {
-            query.append(" WHERE (").append(conditions).append(")");
-        }
-        return query.toString();
+            final String whereClause, Hint hint, boolean escapeCols) {
+        return new QueryBuilder().setFullTableName(fullTableName).setSelectColumns(columns)
+                .setWhereClause(whereClause).setHint(hint).setEscapeCols(escapeCols).build();
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutputTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutputTest.java b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutputTest.java
index c6bb739..a904bca 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutputTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutputTest.java
@@ -47,7 +47,7 @@ public class IndexScrutinyTableOutputTest extends BaseIndexTest {
         String sqlStr =
                 IndexScrutinyTableOutput.getSqlQueryAllInvalidRows(conn, columnNames,
                     SCRUTINY_TIME_MILLIS);
-        assertEquals("SELECT \"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\",\"SOURCE_ROW_PK_HASH\",\"SOURCE_TS\",\"TARGET_TS\",\"HAS_TARGET_ROW\",\"ID\",\"PK_PART2\",\"NAME\",\"ZIP\",\":ID\",\":PK_PART2\",\"0:NAME\",\"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\") IN (('TEST_SCHEMA.TEST_INDEX_COLUMN_NAMES_UTIL','TEST_SCHEMA.TEST_ICN_INDEX',1502908914193))",
+        assertEquals("SELECT \"SOURCE_TABLE\" , \"TARGET_TABLE\" , \"SCRUTINY_EXECUTE_TIME\" , \"SOURCE_ROW_PK_HASH\" , \"SOURCE_TS\" , \"TARGET_TS\" , \"HAS_TARGET_ROW\" , \"ID\" , \"PK_PART2\" , \"NAME\" , \"ZIP\" , \":ID\" , \":PK_PART2\" , \"0:NAME\" , \"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\") IN (('TEST_SCHEMA.TEST_INDEX_COLUMN_NAMES_UTIL','TEST_SCHEMA.TEST_ICN_INDEX',1502908914193))",
             sqlStr);
     }
 
@@ -58,7 +58,7 @@ public class IndexScrutinyTableOutputTest extends BaseIndexTest {
         String query =
                 IndexScrutinyTableOutput.getSqlQueryMissingTargetRows(conn, columnNames,
                     SCRUTINY_TIME_MILLIS);
-        assertEquals("SELECT \"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\",\"SOURCE_ROW_PK_HASH\",\"SOURCE_TS\",\"TARGET_TS\",\"HAS_TARGET_ROW\",\"ID\",\"PK_PART2\",\"NAME\",\"ZIP\",\":ID\",\":PK_PART2\",\"0:NAME\",\"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\", \"HAS_TARGET_ROW\") IN (('TEST_SCHEMA.TEST_INDEX_COLUMN_NAMES_UTIL','TEST_SCHEMA.TEST_ICN_INDEX',1502908914193,false))",
+        assertEquals("SELECT \"SOURCE_TABLE\" , \"TARGET_TABLE\" , \"SCRUTINY_EXECUTE_TIME\" , \"SOURCE_ROW_PK_HASH\" , \"SOURCE_TS\" , \"TARGET_TS\" , \"HAS_TARGET_ROW\" , \"ID\" , \"PK_PART2\" , \"NAME\" , \"ZIP\" , \":ID\" , \":PK_PART2\" , \"0:NAME\" , \"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\", \"HAS_TARGET_ROW\") IN (('TEST_SCHEMA.TEST_INDEX_COLUMN_NAMES_UTIL','TEST_SCHEMA.TEST_ICN_INDEX',1502908914193,false))",
             query);
     }
 
@@ -69,7 +69,7 @@ public class IndexScrutinyTableOutputTest extends BaseIndexTest {
         String query =
                 IndexScrutinyTableOutput.getSqlQueryBadCoveredColVal(conn, columnNames,
                     SCRUTINY_TIME_MILLIS);
-        assertEquals("SELECT \"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\",\"SOURCE_ROW_PK_HASH\",\"SOURCE_TS\",\"TARGET_TS\",\"HAS_TARGET_ROW\",\"ID\",\"PK_PART2\",\"NAME\",\"ZIP\",\":ID\",\":PK_PART2\",\"0:NAME\",\"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\", \"HAS_TARGET_ROW\") IN (('TEST_SCHEMA.TEST_INDEX_COLUMN_NAMES_UTIL','TEST_SCHEMA.TEST_ICN_INDEX',1502908914193,true))",
+        assertEquals("SELECT \"SOURCE_TABLE\" , \"TARGET_TABLE\" , \"SCRUTINY_EXECUTE_TIME\" , \"SOURCE_ROW_PK_HASH\" , \"SOURCE_TS\" , \"TARGET_TS\" , \"HAS_TARGET_ROW\" , \"ID\" , \"PK_PART2\" , \"NAME\" , \"ZIP\" , \":ID\" , \":PK_PART2\" , \"0:NAME\" , \"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\", \"HAS_TARGET_ROW\") IN (('TEST_SCHEMA.TEST_INDEX_COLUMN_NAMES_UTIL','TEST_SCHEMA.TEST_ICN_INDEX',1502908914193,true))",
             query);
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtilTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtilTest.java b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtilTest.java
index f864dd5..0c4c004 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtilTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtilTest.java
@@ -145,7 +145,7 @@ public class PhoenixConfigurationUtilTest extends BaseConnectionlessQueryTest {
             configuration.set(HConstants.ZOOKEEPER_QUORUM, getUrl());
             PhoenixConfigurationUtil.setInputTableName(configuration, tableName);
             final String selectStatement = PhoenixConfigurationUtil.getSelectStatement(configuration);
-            final String expectedSelectStatement = "SELECT \"A_STRING\",\"A_BINARY\",\"0\".\"COL1\" FROM " + tableName ; 
+            final String expectedSelectStatement = "SELECT \"A_STRING\" , \"A_BINARY\" , \"0\".\"COL1\" FROM " + tableName ;
             assertEquals(expectedSelectStatement, selectStatement);
         } finally {
             conn.close();
@@ -167,7 +167,7 @@ public class PhoenixConfigurationUtilTest extends BaseConnectionlessQueryTest {
             configuration.set(HConstants.ZOOKEEPER_QUORUM, getUrl());
             PhoenixConfigurationUtil.setInputTableName(configuration, fullTableName);
             final String selectStatement = PhoenixConfigurationUtil.getSelectStatement(configuration);
-            final String expectedSelectStatement = "SELECT \"A_STRING\",\"A_BINARY\",\"0\".\"COL1\" FROM " + fullTableName; 
+            final String expectedSelectStatement = "SELECT \"A_STRING\" , \"A_BINARY\" , \"0\".\"COL1\" FROM " + fullTableName;
             assertEquals(expectedSelectStatement, selectStatement);
         } finally {
             conn.close();
@@ -209,7 +209,7 @@ public class PhoenixConfigurationUtilTest extends BaseConnectionlessQueryTest {
             PhoenixConfigurationUtil.setSchemaType(configuration, SchemaType.QUERY);
             PhoenixConfigurationUtil.setInputTableName(configuration, tableName);
             final String selectStatement = PhoenixConfigurationUtil.getSelectStatement(configuration);
-            final String expectedSelectStatement = "SELECT \"ID\",\"0\".\"VCARRAY\" FROM " + tableName ; 
+            final String expectedSelectStatement = "SELECT \"ID\" , \"0\".\"VCARRAY\" FROM " + tableName ;
             assertEquals(expectedSelectStatement, selectStatement);
         } finally {
             conn.close();

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/test/java/org/apache/phoenix/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/QueryUtilTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/QueryUtilTest.java
index 2d094f6..8ee8f97 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/QueryUtilTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/QueryUtilTest.java
@@ -66,14 +66,14 @@ public class QueryUtilTest {
     @Test
     public void testConstructSelectStatement() {
         assertEquals(
-                "SELECT \"ID\",\"NAME\" FROM MYTAB",
+                "SELECT \"ID\" , \"NAME\" FROM MYTAB",
                 QueryUtil.constructSelectStatement("MYTAB", ImmutableList.of(ID_COLUMN,NAME_COLUMN),null));
     }
 
     @Test
     public void testConstructSelectStatementWithSchema() {
         assertEquals(
-                "SELECT \"ID\",\"NAME\" FROM A.MYTAB",
+                "SELECT \"ID\" , \"NAME\" FROM A.MYTAB",
                 QueryUtil.constructSelectStatement("A.MYTAB", ImmutableList.of(ID_COLUMN,NAME_COLUMN),null));
     }
     
@@ -83,7 +83,7 @@ public class QueryUtilTest {
         final String schemaName = SchemaUtil.getEscapedArgument("a");
         final String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
         assertEquals(
-                "SELECT \"ID\",\"NAME\" FROM \"a\".MYTAB",
+                "SELECT \"ID\" , \"NAME\" FROM \"a\".MYTAB",
                 QueryUtil.constructSelectStatement(fullTableName, ImmutableList.of(ID_COLUMN,NAME_COLUMN),null));
     }
     
@@ -93,14 +93,14 @@ public class QueryUtilTest {
         final String schemaName = SchemaUtil.getEscapedArgument("a");
         final String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
         assertEquals(
-                "SELECT \"ID\",\"NAME\" FROM \"a\".\"mytab\"",
+                "SELECT \"ID\" , \"NAME\" FROM \"a\".\"mytab\"",
                 QueryUtil.constructSelectStatement(fullTableName, ImmutableList.of(ID_COLUMN,NAME_COLUMN),null));
     }
 
     @Test
     public void testConstructSelectWithHint() {
         assertEquals(
-            "SELECT /*+ NO_INDEX */ \"col1\",\"col2\" FROM MYTAB WHERE (\"col2\"=? and \"col3\" is null)",
+            "SELECT /*+ NO_INDEX */ \"col1\" , \"col2\" FROM MYTAB WHERE (\"col2\"=? and \"col3\" is null)",
             QueryUtil.constructSelectStatement("MYTAB", Lists.newArrayList("col1", "col2"),
                 "\"col2\"=? and \"col3\" is null", Hint.NO_INDEX, true));
     }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-spark/pom.xml b/phoenix-spark/pom.xml
index 602dfb1..d3979f4 100644
--- a/phoenix-spark/pom.xml
+++ b/phoenix-spark/pom.xml
@@ -488,6 +488,14 @@
     <testResources><testResource><directory>src/it/resources</directory></testResource></testResources>
     <plugins>
       <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>build-helper-maven-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-failsafe-plugin</artifactId>
+      </plugin>
+      <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
       </plugin>

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/src/it/java/org/apache/phoenix/spark/AggregateIT.java
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/it/java/org/apache/phoenix/spark/AggregateIT.java b/phoenix-spark/src/it/java/org/apache/phoenix/spark/AggregateIT.java
new file mode 100644
index 0000000..e4b96a3
--- /dev/null
+++ b/phoenix-spark/src/it/java/org/apache/phoenix/spark/AggregateIT.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.spark;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.apache.phoenix.end2end.BaseAggregateIT;
+import org.apache.phoenix.util.QueryBuilder;
+
+public class AggregateIT extends BaseAggregateIT {
+
+    @Override
+    protected ResultSet executeQueryThrowsException(Connection conn, QueryBuilder queryBuilder,
+        String expectedPhoenixExceptionMsg, String expectedSparkExceptionMsg) {
+        ResultSet rs = null;
+        try {
+            rs = executeQuery(conn, queryBuilder);
+            fail();
+        }
+        catch(Exception e) {
+            assertTrue(e.getMessage().contains(expectedSparkExceptionMsg));
+        }
+        return rs;
+    }
+
+    @Override
+    protected ResultSet executeQuery(Connection conn, QueryBuilder queryBuilder) throws SQLException {
+        return SparkUtil.executeQuery(conn, queryBuilder, getUrl(), config);
+    }
+
+    @Override
+    protected void testCountNullInNonEmptyKeyValueCF(int columnEncodedBytes) throws Exception {
+        try (Connection conn = DriverManager.getConnection(getUrl())) {
+            //Type is INT
+            String intTableName=generateUniqueName();
+            String sql="create table " + intTableName + " (mykey integer not null primary key, A.COLA integer, B.COLB integer) "
+                    + "IMMUTABLE_ROWS=true, IMMUTABLE_STORAGE_SCHEME = ONE_CELL_PER_COLUMN, COLUMN_ENCODED_BYTES = " + columnEncodedBytes + ", DISABLE_WAL=true";
+
+            conn.createStatement().execute(sql);
+            conn.createStatement().execute("UPSERT INTO "+intTableName+" VALUES (1,1)");
+            conn.createStatement().execute("UPSERT INTO "+intTableName+" VALUES (2,1)");
+            conn.createStatement().execute("UPSERT INTO "+intTableName+" VALUES (3,1,2)");
+            conn.createStatement().execute("UPSERT INTO "+intTableName+" VALUES (4,1)");
+            conn.createStatement().execute("UPSERT INTO "+intTableName+" VALUES (5,1)");
+            conn.commit();
+
+            sql="select count(*) from "+intTableName;
+            QueryBuilder queryBuilder = new QueryBuilder()
+                .setSelectExpression("COUNT(*)")
+                .setFullTableName(intTableName);
+            ResultSet rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertEquals(5, rs.getLong(1));
+
+            sql="select count(*) from "+intTableName + " where b.colb is not null";
+            queryBuilder.setWhereClause("`B.COLB` IS NOT NULL");
+            rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertEquals(1, rs.getLong(1));
+
+            sql="select count(*) from "+intTableName + " where b.colb is null";
+            queryBuilder.setWhereClause("`B.COLB` IS NULL");
+            rs = executeQuery(conn, queryBuilder);
+            assertTrue(rs.next());
+            assertEquals(4, rs.getLong(1));
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/src/it/java/org/apache/phoenix/spark/OrderByIT.java
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/it/java/org/apache/phoenix/spark/OrderByIT.java b/phoenix-spark/src/it/java/org/apache/phoenix/spark/OrderByIT.java
new file mode 100644
index 0000000..bdffaf5
--- /dev/null
+++ b/phoenix-spark/src/it/java/org/apache/phoenix/spark/OrderByIT.java
@@ -0,0 +1,460 @@
+package org.apache.phoenix.spark;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.phoenix.end2end.BaseOrderByIT;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryBuilder;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SQLContext;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+import scala.Option;
+import scala.collection.JavaConverters;
+
+public class OrderByIT extends BaseOrderByIT {
+
+    @Override
+    protected ResultSet executeQueryThrowsException(Connection conn, QueryBuilder queryBuilder,
+                                                    String expectedPhoenixExceptionMsg, String expectedSparkExceptionMsg) {
+        ResultSet rs = null;
+        try {
+            rs = executeQuery(conn, queryBuilder);
+            fail();
+        }
+        catch(Exception e) {
+            assertTrue(e.getMessage().contains(expectedSparkExceptionMsg));
+        }
+        return rs;
+    }
+
+    @Override
+    protected ResultSet executeQuery(Connection conn, QueryBuilder queryBuilder) throws SQLException {
+        return SparkUtil.executeQuery(conn, queryBuilder, getUrl(), config);
+    }
+
+    @Test
+    public void testOrderByWithJoin() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+            conn.setAutoCommit(false);
+            String tableName1 = generateUniqueName();
+            String ddl = "CREATE TABLE " + tableName1 +
+                    "  (a_string varchar not null, cf1.a integer, cf1.b varchar, col1 integer, cf2.c varchar, cf2.d integer " +
+                    "  CONSTRAINT pk PRIMARY KEY (a_string))\n";
+            createTestTable(getUrl(), ddl);
+            String dml = "UPSERT INTO " + tableName1 + " VALUES(?,?,?,?,?,?)";
+            PreparedStatement stmt = conn.prepareStatement(dml);
+            stmt.setString(1, "a");
+            stmt.setInt(2, 40);
+            stmt.setString(3, "aa");
+            stmt.setInt(4, 10);
+            stmt.setString(5, "bb");
+            stmt.setInt(6, 20);
+            stmt.execute();
+            stmt.setString(1, "c");
+            stmt.setInt(2, 30);
+            stmt.setString(3, "cc");
+            stmt.setInt(4, 50);
+            stmt.setString(5, "dd");
+            stmt.setInt(6, 60);
+            stmt.execute();
+            stmt.setString(1, "b");
+            stmt.setInt(2, 40);
+            stmt.setString(3, "bb");
+            stmt.setInt(4, 5);
+            stmt.setString(5, "aa");
+            stmt.setInt(6, 80);
+            stmt.execute();
+            conn.commit();
+
+            String tableName2 = generateUniqueName();
+            ddl = "CREATE TABLE " + tableName2 +
+                    "  (a_string varchar not null, col1 integer" +
+                    "  CONSTRAINT pk PRIMARY KEY (a_string))\n";
+            createTestTable(getUrl(), ddl);
+
+            dml = "UPSERT INTO " + tableName2 + " VALUES(?, ?)";
+            stmt = conn.prepareStatement(dml);
+            stmt.setString(1, "a");
+            stmt.setInt(2, 40);
+            stmt.execute();
+            stmt.setString(1, "b");
+            stmt.setInt(2, 20);
+            stmt.execute();
+            stmt.setString(1, "c");
+            stmt.setInt(2, 30);
+            stmt.execute();
+            conn.commit();
+
+            // create two PhoenixRDDs  using the table names and columns that are required for the JOIN query
+            List<String> table1Columns = Lists.newArrayList("A_STRING", "CF1.A", "CF1.B", "COL1", "CF2.C", "CF2.D");
+            SQLContext sqlContext = SparkUtil.getSqlContext();
+            Dataset phoenixDataSet =
+                    new PhoenixRDD(SparkUtil.getSparkContext(), tableName1,
+                            JavaConverters.collectionAsScalaIterableConverter(table1Columns)
+                                    .asScala().toSeq(),
+                            Option.apply((String) null), Option.apply(getUrl()), config, false,
+                            null).toDataFrame(sqlContext);
+            phoenixDataSet.registerTempTable(tableName1);
+            List<String> table2Columns = Lists.newArrayList("A_STRING", "COL1");
+            phoenixDataSet =
+                    new PhoenixRDD(SparkUtil.getSparkContext(), tableName2,
+                            JavaConverters.collectionAsScalaIterableConverter(table2Columns)
+                                    .asScala().toSeq(),
+                            Option.apply((String) null), Option.apply(getUrl()), config, false,
+                            null).toDataFrame(sqlContext);
+            phoenixDataSet.registerTempTable(tableName2);
+
+            String query =
+                    "SELECT T1.* FROM " + tableName1 + " T1 JOIN " + tableName2
+                            + " T2 ON T1.A_STRING = T2.A_STRING ORDER BY T1.`CF1.B`";
+            Dataset<Row> dataset =
+                    sqlContext.sql(query);
+            List<Row> rows = dataset.collectAsList();
+            ResultSet rs = new SparkResultSet(rows, dataset.columns());
+
+            assertTrue(rs.next());
+            assertEquals("a",rs.getString(1));
+            assertEquals(40,rs.getInt(2));
+            assertEquals("aa",rs.getString(3));
+            assertEquals(10,rs.getInt(4));
+            assertEquals("bb",rs.getString(5));
+            assertEquals(20,rs.getInt(6));
+            assertTrue(rs.next());
+            assertEquals("b",rs.getString(1));
+            assertEquals(40,rs.getInt(2));
+            assertEquals("bb",rs.getString(3));
+            assertEquals(5,rs.getInt(4));
+            assertEquals("aa",rs.getString(5));
+            assertEquals(80,rs.getInt(6));
+            assertTrue(rs.next());
+            assertEquals("c",rs.getString(1));
+            assertEquals(30,rs.getInt(2));
+            assertEquals("cc",rs.getString(3));
+            assertEquals(50,rs.getInt(4));
+            assertEquals("dd",rs.getString(5));
+            assertEquals(60,rs.getInt(6));
+            assertFalse(rs.next());
+
+            query =
+                    "select t1.a_string, t2.col1 from " + tableName1 + " t1 join " + tableName2
+                            + " t2 on t1.a_string = t2.a_string order by t2.col1";
+            dataset =  sqlContext.sql(query);
+            rows = dataset.collectAsList();
+            rs = new SparkResultSet(rows, dataset.columns());
+            assertTrue(rs.next());
+            assertEquals("b",rs.getString(1));
+            assertEquals(20,rs.getInt(2));
+            assertTrue(rs.next());
+            assertEquals("c",rs.getString(1));
+            assertEquals(30,rs.getInt(2));
+            assertTrue(rs.next());
+            assertEquals("a",rs.getString(1));
+            assertEquals(40,rs.getInt(2));
+            assertFalse(rs.next());
+        }
+    }
+
+    @Test
+    public void testOrderByWithUnionAll() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        try (Connection conn = DriverManager.getConnection(getUrl(), props)){
+            conn.setAutoCommit(false);
+            String tableName1 = generateUniqueName();
+            String ddl = "CREATE TABLE  " + tableName1 +
+                    "  (a_string varchar not null, cf1.a integer, cf1.b varchar, col1 integer, cf2.c varchar, cf2.d integer " +
+                    "  CONSTRAINT pk PRIMARY KEY (a_string))\n";
+            createTestTable(getUrl(), ddl);
+            String dml = "UPSERT INTO " + tableName1 + " VALUES(?,?,?,?,?,?)";
+            PreparedStatement stmt = conn.prepareStatement(dml);
+            stmt.setString(1, "a");
+            stmt.setInt(2, 40);
+            stmt.setString(3, "aa");
+            stmt.setInt(4, 10);
+            stmt.setString(5, "bb");
+            stmt.setInt(6, 20);
+            stmt.execute();
+            stmt.setString(1, "c");
+            stmt.setInt(2, 30);
+            stmt.setString(3, "cc");
+            stmt.setInt(4, 50);
+            stmt.setString(5, "dd");
+            stmt.setInt(6, 60);
+            stmt.execute();
+            stmt.setString(1, "b");
+            stmt.setInt(2, 40);
+            stmt.setString(3, "bb");
+            stmt.setInt(4, 5);
+            stmt.setString(5, "aa");
+            stmt.setInt(6, 80);
+            stmt.execute();
+            conn.commit();
+
+            String tableName2 = generateUniqueName();
+            ddl = "CREATE TABLE " + tableName2 +
+                    "  (a_string varchar not null, col1 integer" +
+                    "  CONSTRAINT pk PRIMARY KEY (a_string))\n";
+            createTestTable(getUrl(), ddl);
+
+            dml = "UPSERT INTO " + tableName2 + " VALUES(?, ?)";
+            stmt = conn.prepareStatement(dml);
+            stmt.setString(1, "aa");
+            stmt.setInt(2, 40);
+            stmt.execute();
+            stmt.setString(1, "bb");
+            stmt.setInt(2, 10);
+            stmt.execute();
+            stmt.setString(1, "cc");
+            stmt.setInt(2, 30);
+            stmt.execute();
+            conn.commit();
+
+
+            List<String> table1Columns = Lists.newArrayList("A_STRING", "CF1.A", "CF1.B", "COL1", "CF2.C", "CF2.D");
+            SQLContext sqlContext = SparkUtil.getSqlContext();
+            Dataset phoenixDataSet =
+                    new PhoenixRDD(SparkUtil.getSparkContext(), tableName1,
+                            JavaConverters.collectionAsScalaIterableConverter(table1Columns)
+                                    .asScala().toSeq(),
+                            Option.apply((String) null), Option.apply(getUrl()), config, false,
+                            null).toDataFrame(sqlContext);
+            phoenixDataSet.registerTempTable(tableName1);
+            List<String> table2Columns = Lists.newArrayList("A_STRING", "COL1");
+            phoenixDataSet =
+                    new PhoenixRDD(SparkUtil.getSparkContext(), tableName2,
+                            JavaConverters.collectionAsScalaIterableConverter(table2Columns)
+                                    .asScala().toSeq(),
+                            Option.apply((String) null), Option.apply(getUrl()), config, false,
+                            null).toDataFrame(sqlContext);
+            phoenixDataSet.registerTempTable(tableName2);
+
+            String query =
+                    "select a_string, `cf2.d` from " + tableName1 + " union all select * from "
+                            + tableName2 + " order by `cf2.d`";
+            Dataset<Row> dataset =
+                    sqlContext.sql(query);
+            List<Row> rows = dataset.collectAsList();
+            ResultSet rs = new SparkResultSet(rows, dataset.columns());
+            assertTrue(rs.next());
+            assertEquals("bb",rs.getString(1));
+            assertEquals(10,rs.getInt(2));
+            assertTrue(rs.next());
+            assertEquals("a",rs.getString(1));
+            assertEquals(20,rs.getInt(2));
+            assertTrue(rs.next());
+            assertEquals("cc",rs.getString(1));
+            assertEquals(30,rs.getInt(2));
+            assertTrue(rs.next());
+            assertEquals("aa",rs.getString(1));
+            assertEquals(40,rs.getInt(2));
+            assertTrue(rs.next());
+            assertEquals("c",rs.getString(1));
+            assertEquals(60,rs.getInt(2));
+            assertTrue(rs.next());
+            assertEquals("b",rs.getString(1));
+            assertEquals(80,rs.getInt(2));
+            assertFalse(rs.next());
+        }
+    }
+
+    @Test
+    public void testOrderByWithExpression() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        conn.setAutoCommit(false);
+
+        try {
+            String tableName = generateUniqueName();
+            String ddl = "CREATE TABLE " + tableName +
+                    "  (a_string varchar not null, col1 integer, col2 integer, col3 timestamp, col4 varchar" +
+                    "  CONSTRAINT pk PRIMARY KEY (a_string))\n";
+            createTestTable(getUrl(), ddl);
+
+            Date date = new Date(System.currentTimeMillis());
+            String dml = "UPSERT INTO " + tableName + " VALUES(?, ?, ?, ?, ?)";
+            PreparedStatement stmt = conn.prepareStatement(dml);
+            stmt.setString(1, "a");
+            stmt.setInt(2, 40);
+            stmt.setInt(3, 20);
+            stmt.setDate(4, new Date(date.getTime()));
+            stmt.setString(5, "xxyy");
+            stmt.execute();
+            stmt.setString(1, "b");
+            stmt.setInt(2, 50);
+            stmt.setInt(3, 30);
+            stmt.setDate(4, new Date(date.getTime()-500));
+            stmt.setString(5, "yyzz");
+            stmt.execute();
+            stmt.setString(1, "c");
+            stmt.setInt(2, 60);
+            stmt.setInt(3, 20);
+            stmt.setDate(4, new Date(date.getTime()-300));
+            stmt.setString(5, "ddee");
+            stmt.execute();
+            conn.commit();
+
+            SQLContext sqlContext = SparkUtil.getSqlContext();
+            Dataset phoenixDataSet =
+                    new PhoenixRDD(SparkUtil.getSparkContext(), tableName,
+                            JavaConverters
+                                    .collectionAsScalaIterableConverter(
+                                        Lists.newArrayList("col1", "col2", "col4"))
+                                    .asScala().toSeq(),
+                            Option.apply((String) null), Option.apply(getUrl()), config, false,
+                            null).toDataFrame(sqlContext);
+
+            phoenixDataSet.registerTempTable(tableName);
+            Dataset<Row> dataset =
+                    sqlContext.sql("SELECT col1+col2, col4, a_string FROM " + tableName
+                            + " ORDER BY col1+col2, col4");
+            List<Row> rows = dataset.collectAsList();
+            ResultSet rs = new SparkResultSet(rows, dataset.columns());
+            assertTrue(rs.next());
+            assertEquals("a", rs.getString(3));
+            assertTrue(rs.next());
+            assertEquals("c", rs.getString(3));
+            assertTrue(rs.next());
+            assertEquals("b", rs.getString(3));
+            assertFalse(rs.next());
+        } catch (SQLException e) {
+        } finally {
+            conn.close();
+        }
+    }
+
+    @Test
+    public void testColumnFamily() throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+            conn.setAutoCommit(false);
+            String tableName = generateUniqueName();
+            String ddl = "CREATE TABLE " + tableName +
+                    "  (a_string varchar not null, cf1.a integer, cf1.b varchar, col1 integer, cf2.c varchar, cf2.d integer, col2 integer" +
+                    "  CONSTRAINT pk PRIMARY KEY (a_string))\n";
+            createTestTable(getUrl(), ddl);
+            String dml = "UPSERT INTO " + tableName + " VALUES(?,?,?,?,?,?,?)";
+            PreparedStatement stmt = conn.prepareStatement(dml);
+            stmt.setString(1, "a");
+            stmt.setInt(2, 40);
+            stmt.setString(3, "aa");
+            stmt.setInt(4, 10);
+            stmt.setString(5, "bb");
+            stmt.setInt(6, 20);
+            stmt.setInt(7, 1);
+            stmt.execute();
+            stmt.setString(1, "c");
+            stmt.setInt(2, 30);
+            stmt.setString(3, "cc");
+            stmt.setInt(4, 50);
+            stmt.setString(5, "dd");
+            stmt.setInt(6, 60);
+            stmt.setInt(7, 3);
+            stmt.execute();
+            stmt.setString(1, "b");
+            stmt.setInt(2, 40);
+            stmt.setString(3, "bb");
+            stmt.setInt(4, 5);
+            stmt.setString(5, "aa");
+            stmt.setInt(6, 80);
+            stmt.setInt(7, 2);
+            stmt.execute();
+            conn.commit();
+
+
+            List<String> columns =
+                    Lists.newArrayList("A_STRING", "CF1.A", "CF1.B", "COL1", "CF2.C", "CF2.D",
+                        "COL2");
+
+            SQLContext sqlContext = SparkUtil.getSqlContext();
+            Dataset phoenixDataSet =
+                    new PhoenixRDD(SparkUtil.getSparkContext(), tableName,
+                            JavaConverters.collectionAsScalaIterableConverter(columns).asScala()
+                                    .toSeq(),
+                            Option.apply((String) null), Option.apply(url), config, false, null)
+                                    .toDataFrame(sqlContext);
+
+            phoenixDataSet.registerTempTable(tableName);
+            Dataset<Row> dataset =
+                    sqlContext.sql("SELECT A_STRING, `CF1.A`, `CF1.B`, COL1, `CF2.C`, `CF2.D`, COL2 from "
+                            + tableName + " ORDER BY `CF1.A`,`CF2.C`");
+            List<Row> rows = dataset.collectAsList();
+            ResultSet rs = new SparkResultSet(rows, dataset.columns());
+
+            assertTrue(rs.next());
+            assertEquals("c",rs.getString(1));
+            assertEquals(30,rs.getInt(2));
+            assertEquals("cc",rs.getString(3));
+            assertEquals(50,rs.getInt(4));
+            assertEquals("dd",rs.getString(5));
+            assertEquals(60,rs.getInt(6));
+            assertEquals(3,rs.getInt(7));
+            assertTrue(rs.next());
+            assertEquals("b",rs.getString(1));
+            assertEquals(40,rs.getInt(2));
+            assertEquals("bb",rs.getString(3));
+            assertEquals(5,rs.getInt(4));
+            assertEquals("aa",rs.getString(5));
+            assertEquals(80,rs.getInt(6));
+            assertEquals(2,rs.getInt(7));
+            assertTrue(rs.next());
+            assertEquals("a",rs.getString(1));
+            assertEquals(40,rs.getInt(2));
+            assertEquals("aa",rs.getString(3));
+            assertEquals(10,rs.getInt(4));
+            assertEquals("bb",rs.getString(5));
+            assertEquals(20,rs.getInt(6));
+            assertEquals(1,rs.getInt(7));
+            assertFalse(rs.next());
+
+            dataset =
+                    sqlContext.sql("SELECT A_STRING, `CF1.A`, `CF1.B`, COL1, `CF2.C`, `CF2.D`, COL2 from "
+                            + tableName + " ORDER BY COL2");
+            rows = dataset.collectAsList();
+            rs = new SparkResultSet(rows, dataset.columns());
+
+            assertTrue(rs.next());
+            assertEquals("a",rs.getString(1));
+            assertEquals(40,rs.getInt(2));
+            assertEquals("aa",rs.getString(3));
+            assertEquals(10,rs.getInt(4));
+            assertEquals("bb",rs.getString(5));
+            assertEquals(20,rs.getInt(6));
+            assertEquals(1,rs.getInt(7));
+            assertTrue(rs.next());
+            assertEquals("b",rs.getString(1));
+            assertEquals(40,rs.getInt(2));
+            assertEquals("bb",rs.getString(3));
+            assertEquals(5,rs.getInt(4));
+            assertEquals("aa",rs.getString(5));
+            assertEquals(80,rs.getInt(6));
+            assertEquals(2,rs.getInt(7));
+            assertTrue(rs.next());
+            assertEquals("c",rs.getString(1));
+            assertEquals(30,rs.getInt(2));
+            assertEquals("cc",rs.getString(3));
+            assertEquals(50,rs.getInt(4));
+            assertEquals("dd",rs.getString(5));
+            assertEquals(60,rs.getInt(6));
+            assertEquals(3,rs.getInt(7));
+            assertFalse(rs.next());
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java b/phoenix-spark/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java
new file mode 100644
index 0000000..d72acbd
--- /dev/null
+++ b/phoenix-spark/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.spark;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.apache.phoenix.end2end.salted.BaseSaltedTableIT;
+import org.apache.phoenix.util.QueryBuilder;
+
+public class SaltedTableIT extends BaseSaltedTableIT {
+
+    @Override
+    protected ResultSet executeQueryThrowsException(Connection conn, QueryBuilder queryBuilder,
+                                                    String expectedPhoenixExceptionMsg, String expectedSparkExceptionMsg) {
+        ResultSet rs = null;
+        try {
+            rs = executeQuery(conn, queryBuilder);
+            fail();
+        }
+        catch(Exception e) {
+            assertTrue(e.getMessage().contains(expectedSparkExceptionMsg));
+        }
+        return rs;
+    }
+
+    @Override
+    protected ResultSet executeQuery(Connection conn, QueryBuilder queryBuilder) throws SQLException {
+        return SparkUtil.executeQuery(conn, queryBuilder, getUrl(), config);
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/src/it/java/org/apache/phoenix/spark/SparkUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/it/java/org/apache/phoenix/spark/SparkUtil.java b/phoenix-spark/src/it/java/org/apache/phoenix/spark/SparkUtil.java
new file mode 100644
index 0000000..6285209
--- /dev/null
+++ b/phoenix-spark/src/it/java/org/apache/phoenix/spark/SparkUtil.java
@@ -0,0 +1,87 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.spark;
+
+import com.google.common.base.Joiner;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.QueryBuilder;
+import org.apache.spark.SparkContext;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.execution.SparkPlan;
+import scala.Option;
+import scala.collection.JavaConverters;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+
+public class SparkUtil {
+
+    public static final String APP_NAME = "Java Spark Tests";
+    public static final String NUM_EXECUTORS = "local[2]";
+    public static final String UI_SHOW_CONSOLE_PROGRESS = "spark.ui.showConsoleProgress";
+
+    public static SparkContext getSparkContext() {
+        return SparkSession.builder().appName(APP_NAME).master(NUM_EXECUTORS)
+                .config(UI_SHOW_CONSOLE_PROGRESS, false).getOrCreate().sparkContext();
+    }
+
+    public static SQLContext getSqlContext() {
+        return SparkSession.builder().appName(APP_NAME).master(NUM_EXECUTORS)
+                .config(UI_SHOW_CONSOLE_PROGRESS, false).getOrCreate().sqlContext();
+    }
+
+    public static ResultSet executeQuery(Connection conn, QueryBuilder queryBuilder, String url, Configuration config)
+            throws SQLException {
+        SQLContext sqlContext = SparkUtil.getSqlContext();
+
+        boolean forceRowKeyOrder =
+                conn.unwrap(PhoenixConnection.class).getQueryServices().getProps()
+                        .getBoolean(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, false);
+        // if we are forcing row key order we have to add an ORDER BY
+        // here we assume that the required columns are in the primary key column order
+        String prevOrderBy = queryBuilder.getOrderByClause();
+        if (forceRowKeyOrder &&  (queryBuilder.getOrderByClause()==null || queryBuilder.getOrderByClause().isEmpty())) {
+            queryBuilder.setOrderByClause(Joiner.on(", ").join(queryBuilder.getRequiredColumns()));
+        }
+
+        // create PhoenixRDD using the table name and columns that are required by the query
+        // since we don't set the predicate filtering is done after rows are returned from spark
+        Dataset phoenixDataSet =
+                new PhoenixRDD(SparkUtil.getSparkContext(), queryBuilder.getFullTableName(),
+                        JavaConverters.collectionAsScalaIterableConverter(queryBuilder.getRequiredColumns()).asScala()
+                                .toSeq(),
+                        Option.apply((String) null), Option.apply(url), config, false,
+                        null).toDataFrame(sqlContext);
+
+        phoenixDataSet.registerTempTable(queryBuilder.getFullTableName());
+        Dataset<Row> dataset = sqlContext.sql(queryBuilder.build());
+        SparkPlan plan = dataset.queryExecution().executedPlan();
+        List<Row> rows = dataset.collectAsList();
+        queryBuilder.setOrderByClause(prevOrderBy);
+        ResultSet rs = new SparkResultSet(rows, dataset.columns());
+        return rs;
+    }
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala b/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
index 4e11acc..d1e38fa 100644
--- a/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
+++ b/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
@@ -23,6 +23,7 @@ import org.joda.time.DateTime
 import org.apache.spark.{SparkConf, SparkContext}
 import scala.collection.mutable.ListBuffer
 import org.apache.hadoop.conf.Configuration
+
 /**
   * Note: If running directly from an IDE, these are the recommended VM parameters:
   * -Xmx1536m -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m
@@ -287,11 +288,11 @@ class PhoenixSparkIT extends AbstractPhoenixSparkIT {
 
     val plan = res.queryExecution.sparkPlan
     // filters should be pushed into phoenix relation
-    assert(plan.toString.contains("PushedFilters: [IsNotNull(COL1), IsNotNull(ID), " +
-      "EqualTo(COL1,test_row_1), EqualTo(ID,1)]"))
+    assert(plan.toString.contains("PushedFilters: [*IsNotNull(COL1), *IsNotNull(ID), " +
+      "*EqualTo(COL1,test_row_1), *EqualTo(ID,1)]"))
     // spark should run the filters on the rows returned by Phoenix
-    assert(!plan.toString.contains("Filter (((isnotnull(COL1#8) && isnotnull(ID#7L)) " +
-      "&& (COL1#8 = test_row_1)) && (ID#7L = 1))"))
+    assert(!plan.toString.matches(".*Filter (((isnotnull(COL1.*) && isnotnull(ID.*)) "
+      + " && (COL1.* = test_row_1)) && (ID.* = 1)).*"))
   }
 
   test("Can persist a dataframe using 'DataFrame.saveToPhoenix'") {