You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by td...@apache.org on 2018/05/03 03:33:18 UTC

[01/50] [abbrv] phoenix git commit: PHOENIX-4669 NoSuchColumnFamilyException when creating index on views that are built on tables which have named column family

Repository: phoenix
Updated Branches:
  refs/heads/system-catalog 15d4dace4 -> f7d87ce2f


PHOENIX-4669 NoSuchColumnFamilyException when creating index on views that are built on tables which have named column family

Signed-off-by: ss77892 <ss...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/98a8ddda
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/98a8ddda
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/98a8ddda

Branch: refs/heads/system-catalog
Commit: 98a8ddda49eb38b72984ea3800c4ed37299f0254
Parents: 28c11fe
Author: Toshihiro Suzuki <br...@gmail.com>
Authored: Thu Mar 29 17:17:37 2018 +0900
Committer: ss77892 <ss...@apache.org>
Committed: Mon Apr 9 21:39:40 2018 -0700

----------------------------------------------------------------------
 .../phoenix/end2end/index/ViewIndexIT.java      | 45 +++++++++++++++++++-
 .../query/ConnectionQueryServicesImpl.java      | 27 ++++++------
 2 files changed, 57 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/98a8ddda/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
index 53bb550..8ffd798 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
@@ -31,6 +31,7 @@ import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.sql.Statement;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
@@ -443,5 +444,47 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
         assertFalse(rs.next());
         assertEquals(indexName, stmt.getQueryPlan().getContext().getCurrentTable().getTable().getName().getString());
     }
-    
+
+    @Test
+    public void testCreatingIndexOnViewBuiltOnTableWithOnlyNamedColumnFamilies() throws Exception {
+        try (Connection c = getConnection(); Statement s = c.createStatement()) {
+            String tableName = generateUniqueName();
+            String viewName = generateUniqueName();
+            String indexName = generateUniqueName();
+
+            c.setAutoCommit(true);
+            s.execute("CREATE TABLE " + tableName + " (COL1 VARCHAR PRIMARY KEY, CF.COL2 VARCHAR)");
+            s.executeUpdate("UPSERT INTO " + tableName + " VALUES ('AAA', 'BBB')");
+            s.execute("CREATE VIEW " + viewName + " AS SELECT * FROM " + tableName);
+            s.execute("CREATE INDEX " + indexName + " ON " + viewName + " (CF.COL2)");
+
+            try (ResultSet rs = s.executeQuery("SELECT * FROM " + viewName + " WHERE CF.COL2 = 'BBB'")) {
+                assertTrue(rs.next());
+                assertEquals("AAA", rs.getString("COL1"));
+                assertEquals("BBB", rs.getString("COL2"));
+            }
+        }
+        try (Connection c = getConnection(); Statement s = c.createStatement()) {
+            String tableName = generateUniqueName();
+            String viewName = generateUniqueName();
+            String index1Name = generateUniqueName();
+            String index2Name = generateUniqueName();
+
+            c.setAutoCommit(true);
+            s.execute("create table " + tableName + " (i1 integer primary key, c2.i2 integer, c3.i3 integer, c4.i4 integer)");
+            s.execute("create view " + viewName + " as select * from " + tableName + " where c2.i2 = 1");
+            s.executeUpdate("upsert into " + viewName + "(i1, c3.i3, c4.i4) VALUES (1, 1, 1)");
+            s.execute("create index " + index1Name + " ON " + viewName + " (c3.i3)");
+            s.execute("create index " + index2Name + " ON " + viewName + " (c3.i3) include (c4.i4)");
+            s.executeUpdate("upsert into " + viewName + "(i1, c3.i3, c4.i4) VALUES (2, 2, 2)");
+
+            try (ResultSet rs = s.executeQuery("select * from " + viewName + " WHERE c3.i3 = 1")) {
+                assertTrue(rs.next());
+                assertEquals(1, rs.getInt("i1"));
+                assertEquals(1, rs.getInt("i2"));
+                assertEquals(1, rs.getInt("i3"));
+                assertEquals(1, rs.getInt("i4"));
+            }
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/98a8ddda/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index eff406d..6df2f80 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -1690,22 +1690,20 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
             throws SQLException {
         byte[] physicalTableName = table.getPhysicalName().getBytes();
         HTableDescriptor htableDesc = this.getTableDescriptor(physicalTableName);
-        Map<String,Object> tableProps = createPropertiesMap(htableDesc.getValues());
-        List<Pair<byte[],Map<String,Object>>> families = Lists.newArrayListWithExpectedSize(Math.max(1, table.getColumnFamilies().size()+1));
-        if (families.isEmpty()) {
-            byte[] familyName = SchemaUtil.getEmptyColumnFamily(table);
+        List<Pair<byte[],Map<String,Object>>> families = Lists.newArrayListWithExpectedSize(Math.max(1, table.getColumnFamilies().size() + 1));
+
+        // Create all column families that the parent table has
+        for (PColumnFamily family : table.getColumnFamilies()) {
+            byte[] familyName = family.getName().getBytes();
             Map<String,Object> familyProps = createPropertiesMap(htableDesc.getFamily(familyName).getValues());
-            families.add(new Pair<byte[],Map<String,Object>>(familyName, familyProps));
-        } else {
-            for (PColumnFamily family : table.getColumnFamilies()) {
-                byte[] familyName = family.getName().getBytes();
-                Map<String,Object> familyProps = createPropertiesMap(htableDesc.getFamily(familyName).getValues());
-                families.add(new Pair<byte[],Map<String,Object>>(familyName, familyProps));
-            }
-            // Always create default column family, because we don't know in advance if we'll
-            // need it for an index with no covered columns.
-            families.add(new Pair<byte[],Map<String,Object>>(table.getDefaultFamilyName().getBytes(), Collections.<String,Object>emptyMap()));
+            families.add(new Pair<>(familyName, familyProps));
         }
+        // Always create default column family, because we don't know in advance if we'll
+        // need it for an index with no covered columns.
+        byte[] defaultFamilyName = table.getDefaultFamilyName() == null ?
+          QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES : table.getDefaultFamilyName().getBytes();
+        families.add(new Pair<>(defaultFamilyName, Collections.<String,Object>emptyMap()));
+
         byte[][] splits = null;
         if (table.getBucketNum() != null) {
             splits = SaltingUtil.getSalteByteSplitPoints(table.getBucketNum());
@@ -1713,6 +1711,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
 
         // Transfer over table values into tableProps
         // TODO: encapsulate better
+        Map<String,Object> tableProps = createPropertiesMap(htableDesc.getValues());
         tableProps.put(PhoenixDatabaseMetaData.TRANSACTIONAL, table.isTransactional());
         tableProps.put(PhoenixDatabaseMetaData.IMMUTABLE_ROWS, table.isImmutableRows());
         ensureViewIndexTableCreated(physicalTableName, tableProps, families, splits, timestamp, isNamespaceMapped);


[39/50] [abbrv] phoenix git commit: PHOENIX-4700 Fix split policy on system tables other than SYSTEM.CATALOG

Posted by td...@apache.org.
PHOENIX-4700 Fix split policy on system tables other than SYSTEM.CATALOG


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/5c637e6d
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/5c637e6d
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/5c637e6d

Branch: refs/heads/system-catalog
Commit: 5c637e6dd4aabb9d8afe0e07a81533a96e111bfb
Parents: 8dd637d
Author: James Taylor <jt...@salesforce.com>
Authored: Mon Apr 23 10:14:36 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Thu Apr 26 09:55:28 2018 -0700

----------------------------------------------------------------------
 .../apache/phoenix/query/QueryConstants.java    | 117 +------------------
 .../SplitOnLeadingVarCharColumnsPolicy.java     |  43 +++++++
 .../schema/SystemFunctionSplitPolicy.java       |  27 +++++
 .../phoenix/schema/SystemStatsSplitPolicy.java  |  27 +++++
 .../phoenix/schema/SystemSplitPolicyTest.java   |  97 +++++++++++++++
 5 files changed, 200 insertions(+), 111 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/5c637e6d/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index 22fa2f4..4285334 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -18,112 +18,7 @@
 package org.apache.phoenix.query;
 
 
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.AUTO_PARTITION_SEQ;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BASE_COLUMN_COUNT;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BUFFER_LENGTH;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CACHE_SIZE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CHAR_OCTET_LENGTH;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLASS_NAME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CURRENT_VALUE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CYCLE_FLAG;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ENCODING_SCHEME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POST_KEY;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INCREMENT_BY;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_ARRAY;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_AUTOINCREMENT;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_CONSTANT;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_NAMESPACE_MAPPED;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_NULLABLE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_ROW_TIMESTAMP;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_VIEW_REFERENCED;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.JAR_PATH;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.KEY_SEQ;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LAST_STATS_UPDATE_TIME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LIMIT_REACHED_FLAG;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MAX_VALUE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MIN_VALUE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MULTI_TENANT;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NULLABLE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_ARGS;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_PREC_RADIX;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PHYSICAL_NAME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PK_NAME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.REF_GENERATION;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.REMARKS;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.RETURN_TYPE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SALT_BUCKETS;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_CATALOG;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_SCHEMA;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_TABLE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SELF_REFERENCING_COL_NAME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SEQUENCE_NAME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SEQUENCE_SCHEMA;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SOURCE_DATA_TYPE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATA_TYPE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATETIME_SUB;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_WITH;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STORE_NULLS;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_TABLE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTION_PROVIDER;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_NAME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BIND_PARAMETERS;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLIENT_IP;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXCEPTION_TRACE;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXPLAIN_PLAN;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GLOBAL_SCAN_DETAILS;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NO_OF_RESULTS_ITERATED;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_ID;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_STATUS;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCAN_METRICS_JSON;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_TIME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TOTAL_EXECUTION_TIME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USER;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.*;
 
 import java.math.BigDecimal;
 
@@ -139,6 +34,8 @@ import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PNameFactory;
 import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.SystemFunctionSplitPolicy;
+import org.apache.phoenix.schema.SystemStatsSplitPolicy;
 import org.apache.phoenix.schema.TableProperty;
 
 
@@ -362,7 +259,7 @@ public interface QueryConstants {
             + PHYSICAL_NAME + ","
             + COLUMN_FAMILY + ","+ GUIDE_POST_KEY+"))\n" +
             // Install split policy to prevent a physical table's stats from being split across regions.
-            HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + 
+            HTableDescriptor.SPLIT_POLICY + "='" + SystemStatsSplitPolicy.class.getName() + "',\n" + 
             PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
 
     public static final String CREATE_SEQUENCE_METADATA =
@@ -406,7 +303,7 @@ public interface QueryConstants {
             HConstants.VERSIONS + "=%s,\n" +
             HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n"+
             // Install split policy to prevent a tenant's metadata from being split across regions.
-            HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + 
+            HTableDescriptor.SPLIT_POLICY + "='" + SystemFunctionSplitPolicy.class.getName() + "',\n" + 
             PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
     
     public static final String CREATE_LOG_METADATA =
@@ -431,9 +328,7 @@ public interface QueryConstants {
             " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (QUERY_ID))\n" +
             HConstants.VERSIONS + "= " + MetaDataProtocol.DEFAULT_LOG_VERSIONS + ",\n" +
             HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n"+
-            // Install split policy to prevent a tenant's metadata from being split across regions.
-            HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + 
-            PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE+ ",\n" + 
+            PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE+ ",\n" +
             HColumnDescriptor.TTL + "=" + MetaDataProtocol.DEFAULT_LOG_TTL+",\n"+
             TableProperty.COLUMN_ENCODED_BYTES.toString()+" = 0";
     

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5c637e6d/phoenix-core/src/main/java/org/apache/phoenix/schema/SplitOnLeadingVarCharColumnsPolicy.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/SplitOnLeadingVarCharColumnsPolicy.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/SplitOnLeadingVarCharColumnsPolicy.java
new file mode 100644
index 0000000..d481998
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/SplitOnLeadingVarCharColumnsPolicy.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema;
+
+import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
+import org.apache.phoenix.util.SchemaUtil;
+
+public abstract class SplitOnLeadingVarCharColumnsPolicy extends ConstantSizeRegionSplitPolicy {
+    abstract protected int getColumnToSplitAt();
+    
+    protected final byte[] getSplitPoint(byte[] splitPoint) {
+        int offset = SchemaUtil.getVarCharLength(splitPoint, 0, splitPoint.length, getColumnToSplitAt());
+        // Only split between leading columns indicated.
+        if (offset == splitPoint.length) {
+            return splitPoint;
+        }
+        // Otherwise, an attempt is being made to split in the middle of a table.
+        // Just return a split point at the boundary of the first two columns instead
+        byte[] newSplitPoint = new byte[offset + 1];
+        System.arraycopy(splitPoint, 0, newSplitPoint, 0, offset+1);
+        return newSplitPoint;
+    }
+    
+    @Override
+    protected final byte[] getSplitPoint() {
+        return getSplitPoint(super.getSplitPoint());
+    }
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5c637e6d/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemFunctionSplitPolicy.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemFunctionSplitPolicy.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemFunctionSplitPolicy.java
new file mode 100644
index 0000000..58e1f9f
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemFunctionSplitPolicy.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema;
+
+public class SystemFunctionSplitPolicy extends SplitOnLeadingVarCharColumnsPolicy {
+
+    @Override
+    protected int getColumnToSplitAt() {
+        return 2;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5c637e6d/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemStatsSplitPolicy.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemStatsSplitPolicy.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemStatsSplitPolicy.java
new file mode 100644
index 0000000..69fe8aa
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemStatsSplitPolicy.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema;
+
+public class SystemStatsSplitPolicy extends SplitOnLeadingVarCharColumnsPolicy {
+
+    @Override
+    protected int getColumnToSplitAt() {
+        return 1;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5c637e6d/phoenix-core/src/test/java/org/apache/phoenix/schema/SystemSplitPolicyTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/SystemSplitPolicyTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/SystemSplitPolicyTest.java
new file mode 100644
index 0000000..01074b4
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/SystemSplitPolicyTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema;
+
+import static org.junit.Assert.assertArrayEquals;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.stats.StatisticsUtil;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.schema.types.PLong;
+import org.apache.phoenix.schema.types.PVarchar;
+import org.apache.phoenix.util.ByteUtil;
+import org.junit.Test;
+
+public class SystemSplitPolicyTest {
+    @Test
+    public void testStatsSplitPolicy() {
+        SplitOnLeadingVarCharColumnsPolicy policy = new SystemStatsSplitPolicy();
+        byte[] splitOn;
+        byte[] rowKey;
+        byte[] table;
+        ImmutableBytesWritable family;
+        table = PVarchar.INSTANCE.toBytes("FOO.BAR");
+        family = QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES_PTR;
+        rowKey = ByteUtil.concat(
+                PLong.INSTANCE.toBytes(20L), 
+                PVarchar.INSTANCE.toBytes("BAS"), 
+                QueryConstants.SEPARATOR_BYTE_ARRAY, 
+                PInteger.INSTANCE.toBytes(100));
+        splitOn = StatisticsUtil.getRowKey(table, family, rowKey);
+        splitOn = policy.getSplitPoint(splitOn);
+        assertArrayEquals(ByteUtil.concat(table, QueryConstants.SEPARATOR_BYTE_ARRAY), splitOn);
+        
+        table = PVarchar.INSTANCE.toBytes("MY_TABLE");
+        family = new ImmutableBytesWritable(Bytes.toBytes("ABC"));
+        rowKey = ByteUtil.concat(
+                PVarchar.INSTANCE.toBytes("BAS"), 
+                QueryConstants.SEPARATOR_BYTE_ARRAY, 
+                PInteger.INSTANCE.toBytes(100),
+                PLong.INSTANCE.toBytes(20L));
+        splitOn = StatisticsUtil.getRowKey(table, family, rowKey);
+        splitOn = policy.getSplitPoint(splitOn);
+        assertArrayEquals(ByteUtil.concat(table, QueryConstants.SEPARATOR_BYTE_ARRAY), splitOn);
+    }
+    
+    private static byte[] getSystemFunctionRowKey(String tenantId, String funcName, String typeName, byte[] argPos) {
+        return ByteUtil.concat(PVarchar.INSTANCE.toBytes(tenantId), 
+                QueryConstants.SEPARATOR_BYTE_ARRAY,
+                PVarchar.INSTANCE.toBytes(funcName),
+                QueryConstants.SEPARATOR_BYTE_ARRAY,
+                PVarchar.INSTANCE.toBytes(typeName), 
+                QueryConstants.SEPARATOR_BYTE_ARRAY,
+                argPos
+                );
+    }
+    
+    private static byte[] getSystemFunctionSplitKey(String tenantId, String funcName) {
+        return ByteUtil.concat(PVarchar.INSTANCE.toBytes(tenantId), 
+                QueryConstants.SEPARATOR_BYTE_ARRAY,
+                PVarchar.INSTANCE.toBytes(funcName),
+                QueryConstants.SEPARATOR_BYTE_ARRAY);
+    }
+    
+    @Test
+    public void testFunctionSplitPolicy() {
+        SplitOnLeadingVarCharColumnsPolicy policy = new SystemFunctionSplitPolicy();
+        byte[] splitPoint;
+        byte[] rowKey;
+        byte[] expectedSplitPoint;
+        rowKey = getSystemFunctionRowKey("","MY_FUNC", "VARCHAR", Bytes.toBytes(3));
+        expectedSplitPoint = getSystemFunctionSplitKey("","MY_FUNC");
+        splitPoint = policy.getSplitPoint(rowKey);
+        assertArrayEquals(expectedSplitPoint, splitPoint);
+        
+        rowKey = getSystemFunctionRowKey("TENANT1","F", "", Bytes.toBytes(3));
+        expectedSplitPoint = getSystemFunctionSplitKey("TENANT1","F");
+        splitPoint = policy.getSplitPoint(rowKey);
+        assertArrayEquals(expectedSplitPoint, splitPoint);
+    }
+}


[02/50] [abbrv] phoenix git commit: PHOENIX-2715 Query Log (Ankit Singhal)

Posted by td...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/log/TableLogWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/log/TableLogWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/log/TableLogWriter.java
new file mode 100644
index 0000000..c102855
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/log/TableLogWriter.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.log;
+
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.Map.Entry;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.expression.Determinism;
+import org.apache.phoenix.expression.LiteralExpression;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.SchemaUtil;
+
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * Writes RingBuffer log event into table 
+ * 
+ */
+public class TableLogWriter implements LogWriter {
+    private static final Log LOG = LogFactory.getLog(LogWriter.class);
+    private Connection connection;
+    private boolean isClosed;
+    private Table table;
+    private Configuration config;
+
+    public TableLogWriter(Configuration configuration) {
+        this.config = configuration;
+        try {
+            this.connection = ConnectionFactory.createConnection(configuration);
+            table = this.connection.getTable(SchemaUtil.getPhysicalTableName(
+                    SchemaUtil.getTableNameAsBytes(SYSTEM_CATALOG_SCHEMA, SYSTEM_LOG_TABLE), config));
+        } catch (Exception e) {
+            LOG.warn("Unable to initiate LogWriter for writing query logs to table");
+        }
+    }
+
+    @Override
+    public void write(RingBufferEvent event) throws SQLException, IOException {
+        if(isClosed()){
+            LOG.warn("Unable to commit query log as Log committer is already closed");
+            return;
+        }
+        if (table == null || connection == null) {
+            LOG.warn("Unable to commit query log as connection was not initiated ");
+            return;
+        }
+        ImmutableMap<QueryLogInfo, Object> queryInfo=event.getQueryInfo();
+        ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+        Put put =new Put(Bytes.toBytes(event.getQueryId()));
+        for (Entry<QueryLogInfo, Object> entry : queryInfo.entrySet()) {
+            if (entry.getKey().logLevel.ordinal() <= event.getConnectionLogLevel().ordinal()) {
+                LiteralExpression expression = LiteralExpression.newConstant(entry.getValue(), entry.getKey().dataType,
+                        Determinism.ALWAYS);
+                expression.evaluate(null, ptr);
+                put.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, Bytes.toBytes(entry.getKey().columnName),
+                        ByteUtil.copyKeyBytesIfNecessary(ptr));
+            }
+        }
+        
+        if (QueryLogInfo.QUERY_STATUS_I.logLevel.ordinal() <= event.getConnectionLogLevel().ordinal()
+                && (event.getLogState() == QueryLogState.COMPLETED || event.getLogState() == QueryLogState.FAILED)) {
+            LiteralExpression expression = LiteralExpression.newConstant(event.getLogState().toString(),
+                    QueryLogInfo.QUERY_STATUS_I.dataType, Determinism.ALWAYS);
+            expression.evaluate(null, ptr);
+            put.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
+                    Bytes.toBytes(QueryLogInfo.QUERY_STATUS_I.columnName), ByteUtil.copyKeyBytesIfNecessary(ptr));
+        }
+        put.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES);
+        table.put(put);
+        
+    }
+    
+    @Override
+    public void close() throws IOException {
+        if(isClosed()){
+            return;
+        }
+        isClosed=true;
+        try {
+            if (table != null) {
+                table.close();
+            }
+            if (connection != null && !connection.isClosed()) {
+                //It should internally close all the statements
+                connection.close();
+            }
+        } catch (IOException e) {
+            // TODO Ignore?
+        }
+    }
+    
+    public boolean isClosed(){
+        return isClosed;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ReadMetricQueue.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ReadMetricQueue.java b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ReadMetricQueue.java
index 4fd1194..c008635 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ReadMetricQueue.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ReadMetricQueue.java
@@ -1,27 +1,21 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
+ * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by
+ * applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
  */
 package org.apache.phoenix.monitoring;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Queue;
@@ -44,6 +38,8 @@ public class ReadMetricQueue {
 
     private final ConcurrentMap<MetricKey, Queue<CombinableMetric>> metricsMap = new ConcurrentHashMap<>();
 
+    private final List<ScanMetricsHolder> scanMetricsHolderList = new ArrayList<ScanMetricsHolder>();
+
     private final boolean isRequestMetricsEnabled;
 
     public ReadMetricQueue(boolean isRequestMetricsEnabled) {
@@ -85,7 +81,7 @@ public class ReadMetricQueue {
         }
         return publishedMetrics;
     }
-    
+
     public void clearMetrics() {
         metricsMap.clear(); // help gc
     }
@@ -177,8 +173,18 @@ public class ReadMetricQueue {
         return q;
     }
 
-	public boolean isRequestMetricsEnabled() {
-		return isRequestMetricsEnabled;
-	}
+    public boolean isRequestMetricsEnabled() {
+        return isRequestMetricsEnabled;
+    }
+    
+    public void addScanHolder(ScanMetricsHolder holder){
+        scanMetricsHolderList.add(holder);
+    }
+
+    public List<ScanMetricsHolder> getScanMetricsHolderList() {
+        return scanMetricsHolderList;
+    }
+    
+    
 
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ScanMetricsHolder.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ScanMetricsHolder.java b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ScanMetricsHolder.java
index 6bcd402..9125cd8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ScanMetricsHolder.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/monitoring/ScanMetricsHolder.java
@@ -17,20 +17,23 @@
  */
 package org.apache.phoenix.monitoring;
 
-import static org.apache.phoenix.monitoring.MetricType.COUNT_RPC_CALLS;
-import static org.apache.phoenix.monitoring.MetricType.COUNT_REMOTE_RPC_CALLS;
+import static org.apache.phoenix.monitoring.MetricType.COUNT_BYTES_IN_REMOTE_RESULTS;
+import static org.apache.phoenix.monitoring.MetricType.COUNT_BYTES_REGION_SERVER_RESULTS;
 import static org.apache.phoenix.monitoring.MetricType.COUNT_MILLS_BETWEEN_NEXTS;
 import static org.apache.phoenix.monitoring.MetricType.COUNT_NOT_SERVING_REGION_EXCEPTION;
-import static org.apache.phoenix.monitoring.MetricType.COUNT_BYTES_REGION_SERVER_RESULTS;
-import static org.apache.phoenix.monitoring.MetricType.COUNT_BYTES_IN_REMOTE_RESULTS;
+import static org.apache.phoenix.monitoring.MetricType.COUNT_REMOTE_RPC_CALLS;
+import static org.apache.phoenix.monitoring.MetricType.COUNT_REMOTE_RPC_RETRIES;
+import static org.apache.phoenix.monitoring.MetricType.COUNT_ROWS_FILTERED;
+import static org.apache.phoenix.monitoring.MetricType.COUNT_ROWS_SCANNED;
+import static org.apache.phoenix.monitoring.MetricType.COUNT_RPC_CALLS;
+import static org.apache.phoenix.monitoring.MetricType.COUNT_RPC_RETRIES;
 import static org.apache.phoenix.monitoring.MetricType.COUNT_SCANNED_REGIONS;
 
-import org.apache.hadoop.hbase.client.Scan;
+import java.io.IOException;
+import java.util.Map;
 
-import static org.apache.phoenix.monitoring.MetricType.COUNT_RPC_RETRIES;
-import static org.apache.phoenix.monitoring.MetricType.COUNT_REMOTE_RPC_RETRIES;
-import static org.apache.phoenix.monitoring.MetricType.COUNT_ROWS_SCANNED;
-import static org.apache.phoenix.monitoring.MetricType.COUNT_ROWS_FILTERED;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.JsonMapper;
 
 public class ScanMetricsHolder {
 
@@ -45,9 +48,11 @@ public class ScanMetricsHolder {
     private final CombinableMetric countOfRemoteRPCRetries;
     private final CombinableMetric countOfRowsScanned;
     private final CombinableMetric countOfRowsFiltered;
+    private  Map<String, Long> scanMetricMap;
+    private Object scan;
 
     private static final ScanMetricsHolder NO_OP_INSTANCE =
-            new ScanMetricsHolder(new ReadMetricQueue(false), "");
+            new ScanMetricsHolder(new ReadMetricQueue(false), "",null);
 
     public static ScanMetricsHolder getInstance(ReadMetricQueue readMetrics, String tableName,
             Scan scan, boolean isRequestMetricsEnabled) {
@@ -55,10 +60,12 @@ public class ScanMetricsHolder {
             return NO_OP_INSTANCE;
         }
         scan.setScanMetricsEnabled(true);
-        return new ScanMetricsHolder(readMetrics, tableName);
+        return new ScanMetricsHolder(readMetrics, tableName, scan);
     }
 
-    private ScanMetricsHolder(ReadMetricQueue readMetrics, String tableName) {
+    private ScanMetricsHolder(ReadMetricQueue readMetrics, String tableName,Scan scan) {
+        readMetrics.addScanHolder(this);
+        this.scan=scan;
         countOfRPCcalls = readMetrics.allotMetric(COUNT_RPC_CALLS, tableName);
         countOfRemoteRPCcalls = readMetrics.allotMetric(COUNT_REMOTE_RPC_CALLS, tableName);
         sumOfMillisSecBetweenNexts = readMetrics.allotMetric(COUNT_MILLS_BETWEEN_NEXTS, tableName);
@@ -118,4 +125,21 @@ public class ScanMetricsHolder {
         return countOfRowsScanned;
     }
 
+    public Map<String, Long> getScanMetricMap() {
+        return scanMetricMap;
+    }
+
+    public void setScanMetricMap(Map<String, Long> scanMetricMap) {
+        this.scanMetricMap = scanMetricMap;
+    }
+    
+    @Override
+    public String toString() {
+        try {
+            return "{\"scan\":" + scan + ", \"scanMetrics\":" + JsonMapper.writeObjectAsString(scanMetricMap) + "}";
+        } catch (IOException e) {
+            return "{\"Exception while converting scan metrics to Json\":\"" + e.getMessage() + "\"}";
+        }
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index 90f8089..0b72ada 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -30,12 +30,14 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.phoenix.compile.MutationPlan;
 import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
 import org.apache.phoenix.execute.MutationState;
 import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
 import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.log.QueryLoggerDisruptor;
 import org.apache.phoenix.parse.PFunction;
 import org.apache.phoenix.schema.PColumn;
 import org.apache.phoenix.schema.PName;
@@ -146,4 +148,8 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
     void upgradeSystemTables(String url, Properties props) throws SQLException;
     
     public Configuration getConfiguration();
+
+    public User getUser();
+
+    public QueryLoggerDisruptor getQueryDisruptor();
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 6df2f80..8c7441a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -124,6 +124,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
 import org.apache.hadoop.hbase.regionserver.BloomType;
 import org.apache.hadoop.hbase.regionserver.IndexHalfStoreFileReaderGenerator;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.ByteStringer;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
@@ -183,6 +184,7 @@ import org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
+import org.apache.phoenix.log.QueryLoggerDisruptor;
 import org.apache.phoenix.parse.PFunction;
 import org.apache.phoenix.parse.PSchema;
 import org.apache.phoenix.protobuf.ProtobufUtil;
@@ -267,6 +269,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     // don't need.
     private final ReadOnlyProps props;
     private final String userName;
+    private final User user;
     private final ConcurrentHashMap<ImmutableBytesWritable,ConnectionQueryServices> childServices;
     private final GuidePostsCache tableStatsCache;
 
@@ -336,6 +339,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                     return hbaseVersion >= PhoenixDatabaseMetaData.MIN_RENEW_LEASE_VERSION;
                 }
             });
+    private QueryLoggerDisruptor queryDisruptor;
 
     private PMetaData newEmptyMetaData() {
         return new PSynchronizedMetaData(new PMetaDataImpl(INITIAL_META_DATA_TABLE_CAPACITY, getProps()));
@@ -372,6 +376,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         ConfigUtil.setReplicationConfigIfAbsent(this.config);
         this.props = new ReadOnlyProps(this.config.iterator());
         this.userName = connectionInfo.getPrincipal();
+        this.user = connectionInfo.getUser();
         this.latestMetaData = newEmptyMetaData();
         // TODO: should we track connection wide memory usage or just org-wide usage?
         // If connection-wide, create a MemoryManager here, otherwise just use the one from the delegate
@@ -396,6 +401,12 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         this.maxConnectionsAllowed = config.getInt(QueryServices.CLIENT_CONNECTION_MAX_ALLOWED_CONNECTIONS,
             QueryServicesOptions.DEFAULT_CLIENT_CONNECTION_MAX_ALLOWED_CONNECTIONS);
         this.shouldThrottleNumConnections = (maxConnectionsAllowed > 0);
+        try {
+            this.queryDisruptor = new QueryLoggerDisruptor(this.config);
+        } catch (SQLException e) {
+            logger.warn("Unable to initiate qeuery logging service !!");
+            e.printStackTrace();
+        }
 
     }
 
@@ -477,6 +488,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
             }
             closed = true;
             GLOBAL_QUERY_SERVICES_COUNTER.decrement();
+            try {
+                if (this.queryDisruptor != null) {
+                    this.queryDisruptor.close();
+                }
+            } catch (Exception e) {
+                // Ignore
+            }
             SQLException sqlE = null;
             try {
                 // Attempt to return any unused sequences.
@@ -2611,7 +2629,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         try {
             metaConnection.createStatement().execute(QueryConstants.CREATE_FUNCTION_METADATA);
         } catch (TableAlreadyExistsException ignore) {}
-
+        try {
+            metaConnection.createStatement().execute(QueryConstants.CREATE_LOG_METADATA);
+        } catch (TableAlreadyExistsException ignore) {}
         // Catch the IOException to log the error message and then bubble it up for the client to retry.
         try {
             createSysMutexTableIfNotExists(hbaseAdmin, ConnectionQueryServicesImpl.this.getProps());
@@ -2966,6 +2986,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
             try {
                 metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_FUNCTION_METADATA);
             } catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {}
+            try {
+                metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_LOG_METADATA);
+            } catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {}
             ConnectionQueryServicesImpl.this.upgradeRequired.set(false);
             success = true;
         } catch (UpgradeInProgressException | UpgradeNotRequiredException e) {
@@ -4063,6 +4086,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     public String getUserName() {
         return userName;
     }
+    
+    @Override
+    public User getUser() {
+        return user;
+    }
 
     private void checkClosed() {
         if (closed) {
@@ -4488,4 +4516,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     public Configuration getConfiguration() {
         return config;
     }
+
+    @Override
+    public QueryLoggerDisruptor getQueryDisruptor() {
+        return this.queryDisruptor;
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index c510b5a..ad354d1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Addressing;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
@@ -55,6 +56,7 @@ import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
+import org.apache.phoenix.log.QueryLoggerDisruptor;
 import org.apache.phoenix.parse.PFunction;
 import org.apache.phoenix.parse.PSchema;
 import org.apache.phoenix.schema.FunctionNotFoundException;
@@ -112,10 +114,13 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
     private final Map<String, List<HRegionLocation>> tableSplits = Maps.newHashMap();
     private final GuidePostsCache guidePostsCache;
     private final Configuration config;
+
+    private User user;
     
     public ConnectionlessQueryServicesImpl(QueryServices services, ConnectionInfo connInfo, Properties info) {
         super(services);
         userName = connInfo.getPrincipal();
+        user = connInfo.getUser();
         metaData = newEmptyMetaData();
 
         // Use KeyValueBuilder that builds real KeyValues, as our test utils require this
@@ -328,6 +333,9 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
                    metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_FUNCTION_METADATA);
                 } catch (NewerTableAlreadyExistsException ignore) {
                 }
+                try {
+                    metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_LOG_METADATA);
+                } catch (NewerTableAlreadyExistsException ignore) {}
             } catch (SQLException e) {
                 sqlE = e;
             } finally {
@@ -664,4 +672,14 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
     public Configuration getConfiguration() {
         return config;
     }
+
+    @Override
+    public User getUser() {
+        return user;
+    }
+
+    @Override
+    public QueryLoggerDisruptor getQueryDisruptor() {
+        return null;
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index 05d1af6..f5c8a59 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -30,12 +30,14 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.phoenix.compile.MutationPlan;
 import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
 import org.apache.phoenix.execute.MutationState;
 import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
 import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.log.QueryLoggerDisruptor;
 import org.apache.phoenix.parse.PFunction;
 import org.apache.phoenix.parse.PSchema;
 import org.apache.phoenix.schema.PColumn;
@@ -351,4 +353,16 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
     public Configuration getConfiguration() {
         return getDelegate().getConfiguration();
     }
+
+    @Override
+    public User getUser() {
+        return getDelegate().getUser();
+    }
+
+    @Override
+    public QueryLoggerDisruptor getQueryDisruptor() {
+        return getDelegate().getQueryDisruptor();
+    }
+    
+    
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index 7607388..ae12e01 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -109,6 +109,20 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BIND_PARAMETERS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLIENT_IP;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXCEPTION_TRACE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXPLAIN_PLAN;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GLOBAL_SCAN_DETAILS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NO_OF_RESULTS_ITERATED;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_ID;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_STATUS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCAN_METRICS_JSON;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_TIME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TOTAL_EXECUTION_TIME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USER;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE;
 
 import java.math.BigDecimal;
 
@@ -124,6 +138,7 @@ import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PNameFactory;
 import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.TableProperty;
 
 
 /**
@@ -395,10 +410,40 @@ public interface QueryConstants {
             // Install split policy to prevent a tenant's metadata from being split across regions.
             HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + 
             PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
+    
+    public static final String CREATE_LOG_METADATA =
+            "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_LOG_TABLE + "\"(\n" +
+             // Pk columns
+            TENANT_ID + " VARCHAR ," +
+            QUERY_ID + " VARCHAR NOT NULL,\n" +
+            USER + " VARCHAR , \n" +
+            CLIENT_IP + " VARCHAR, \n" +
+            // Function metadata (will be null for argument row)
+            QUERY +  " VARCHAR, \n" +
+            EXPLAIN_PLAN + " VARCHAR, \n" +
+            // Argument metadata (will be null for function row)
+            START_TIME + " TIMESTAMP, \n" +
+            TOTAL_EXECUTION_TIME + " BIGINT, \n" +
+            NO_OF_RESULTS_ITERATED + " BIGINT, \n" +
+            QUERY_STATUS + " VARCHAR, \n" +
+            EXCEPTION_TRACE + " VARCHAR, \n" +
+            GLOBAL_SCAN_DETAILS + " VARCHAR, \n" +
+            BIND_PARAMETERS + " VARCHAR, \n" +
+            SCAN_METRICS_JSON + " VARCHAR, \n" +
+            " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (QUERY_ID))\n" +
+            HConstants.VERSIONS + "= " + MetaDataProtocol.DEFAULT_LOG_VERSIONS + ",\n" +
+            HColumnDescriptor.KEEP_DELETED_CELLS + "="  + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n"+
+            // Install split policy to prevent a tenant's metadata from being split across regions.
+            HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + 
+            PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE+ ",\n" + 
+            HColumnDescriptor.TTL + "=" + MetaDataProtocol.DEFAULT_LOG_TTL+",\n"+
+            TableProperty.COLUMN_ENCODED_BYTES.toString()+" = 0";
+    
     public static final byte[] OFFSET_FAMILY = "f_offset".getBytes();
     public static final byte[] OFFSET_COLUMN = "c_offset".getBytes();
     public static final String LAST_SCAN = "LAST_SCAN";
     public static final byte[] UPGRADE_MUTEX = "UPGRADE_MUTEX".getBytes();
     public static final String HASH_JOIN_CACHE_RETRIES = "hashjoin.client.retries.number";
     public static final int DEFAULT_HASH_JOIN_CACHE_RETRIES = 5;
+    
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 0b18aaa..43b9e5a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -301,6 +301,10 @@ public interface QueryServices extends SQLCloseable {
     // Whether to enable cost-based-decision in the query optimizer
     public static final String COST_BASED_OPTIMIZER_ENABLED = "phoenix.costbased.optimizer.enabled";
     public static final String SMALL_SCAN_THRESHOLD_ATTRIB = "phoenix.query.smallScanThreshold";
+    public static final String LOG_LEVEL = "phoenix.log.level";
+    public static final String LOG_BUFFER_SIZE = "phoenix.log.buffer.size";
+    public static final String LOG_BUFFER_WAIT_STRATEGY = "phoenix.log.wait.strategy";
+    public static final String LOG_SAMPLE_RATE = "phoenix.log.sample.rate";
 
     /**
      * Get executor service used for parallel scans

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 961ab9f..58c9812 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -47,6 +47,8 @@ import static org.apache.phoenix.query.QueryServices.IS_NAMESPACE_MAPPING_ENABLE
 import static org.apache.phoenix.query.QueryServices.IS_SYSTEM_TABLE_MAPPED_TO_NAMESPACE;
 import static org.apache.phoenix.query.QueryServices.KEEP_ALIVE_MS_ATTRIB;
 import static org.apache.phoenix.query.QueryServices.LOCAL_INDEX_CLIENT_UPGRADE_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.LOG_LEVEL;
+import static org.apache.phoenix.query.QueryServices.LOG_SAMPLE_RATE;
 import static org.apache.phoenix.query.QueryServices.MASTER_INFO_PORT_ATTRIB;
 import static org.apache.phoenix.query.QueryServices.MAX_CLIENT_METADATA_CACHE_SIZE_ATTRIB;
 import static org.apache.phoenix.query.QueryServices.MAX_MEMORY_PERC_ATTRIB;
@@ -107,6 +109,7 @@ import org.apache.hadoop.hbase.client.Consistency;
 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
 import org.apache.hadoop.hbase.ipc.controller.ClientRpcControllerFactory;
 import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec;
+import org.apache.phoenix.log.LogLevel;
 import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.PTableRefFactory;
@@ -347,6 +350,8 @@ public class QueryServicesOptions {
     public static final boolean DEFAULT_ENABLE_SERVER_UPSERT_SELECT = false;
 
     public static final boolean DEFAULT_COST_BASED_OPTIMIZER_ENABLED = false;
+    public static final String DEFAULT_LOGGING_LEVEL = LogLevel.OFF.name();
+    public static final String DEFAULT_LOG_SAMPLE_RATE = "1.0";
 
     private final Configuration config;
 
@@ -428,7 +433,9 @@ public class QueryServicesOptions {
             .setIfUnset(USE_STATS_FOR_PARALLELIZATION, DEFAULT_USE_STATS_FOR_PARALLELIZATION)
             .setIfUnset(COST_BASED_OPTIMIZER_ENABLED, DEFAULT_COST_BASED_OPTIMIZER_ENABLED)
             .setIfUnset(UPLOAD_BINARY_DATA_TYPE_ENCODING, DEFAULT_UPLOAD_BINARY_DATA_TYPE_ENCODING)
-            .setIfUnset(PHOENIX_ACLS_ENABLED,  DEFAULT_PHOENIX_ACLS_ENABLED);
+            .setIfUnset(PHOENIX_ACLS_ENABLED,  DEFAULT_PHOENIX_ACLS_ENABLED)
+            .setIfUnset(LOG_LEVEL,  DEFAULT_LOGGING_LEVEL)
+            .setIfUnset(LOG_SAMPLE_RATE,  DEFAULT_LOG_SAMPLE_RATE);
         // HBase sets this to 1, so we reset it to something more appropriate.
         // Hopefully HBase will change this, because we can't know if a user set
         // it to 1, so we'll change it.

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c69b6b1..87c2db6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -77,6 +77,7 @@
     <jackson.version>1.9.2</jackson.version>
     <antlr.version>3.5.2</antlr.version>
     <log4j.version>1.2.17</log4j.version>
+    <disruptor.version>3.3.6</disruptor.version>
     <slf4j.version>1.6.4</slf4j.version>
     <protobuf-java.version>2.5.0</protobuf-java.version>
     <commons-io.version>2.1</commons-io.version>
@@ -952,6 +953,11 @@
         <artifactId>javax.servlet-api</artifactId>
         <version>${servlet.api.version}</version>
       </dependency>
+       <dependency>
+        <groupId>com.lmax</groupId>
+        <artifactId>disruptor</artifactId>
+        <version>${disruptor.version}</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 


[25/50] [abbrv] phoenix git commit: PHOENIX-4298 refactoring to avoid using deprecated API for Put/Delete(Sergey Soldatov)

Posted by td...@apache.org.
PHOENIX-4298 refactoring to avoid using deprecated API for Put/Delete(Sergey Soldatov)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/fc6cf43a
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/fc6cf43a
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/fc6cf43a

Branch: refs/heads/system-catalog
Commit: fc6cf43a476c2048c2ee4431311487b30517a208
Parents: 153b357
Author: Rajeshbabu Chintaguntla <ra...@apache.org>
Authored: Thu Apr 19 17:19:20 2018 +0530
Committer: Rajeshbabu Chintaguntla <ra...@apache.org>
Committed: Thu Apr 19 17:19:20 2018 +0530

----------------------------------------------------------------------
 ...ReplayWithIndexWritesAndCompressedWALIT.java |  2 +-
 .../end2end/ColumnProjectionOptimizationIT.java | 14 ++++----
 .../apache/phoenix/end2end/DynamicColumnIT.java | 12 +++----
 .../apache/phoenix/end2end/DynamicFamilyIT.java | 26 +++++++-------
 .../phoenix/end2end/MappingTableDataTypeIT.java |  4 +--
 .../phoenix/end2end/NativeHBaseTypesIT.java     | 30 ++++++++--------
 .../end2end/QueryDatabaseMetaDataIT.java        |  4 +--
 .../org/apache/phoenix/end2end/UpgradeIT.java   |  4 +--
 .../phoenix/tx/ParameterizedTransactionIT.java  |  4 +--
 .../coprocessor/MetaDataEndpointImpl.java       | 36 ++++++++++----------
 .../UngroupedAggregateRegionObserver.java       |  2 +-
 .../apache/phoenix/index/IndexMaintainer.java   | 12 +++----
 .../query/ConnectionQueryServicesImpl.java      |  2 +-
 .../org/apache/phoenix/schema/PTableImpl.java   |  4 +--
 .../phoenix/schema/stats/StatisticsWriter.java  |  9 +++--
 .../java/org/apache/phoenix/util/IndexUtil.java |  6 ++--
 .../wal/ReadWriteKeyValuesWithCodecTest.java    |  6 ++--
 .../index/covered/CoveredColumnIndexCodec.java  |  2 +-
 .../index/covered/LocalTableStateTest.java      | 10 +++---
 .../covered/TestCoveredColumnIndexCodec.java    |  6 ++--
 .../hbase/index/write/TestIndexWriter.java      | 10 ++----
 .../index/write/TestParalleIndexWriter.java     |  2 +-
 .../write/TestParalleWriterIndexCommitter.java  |  2 +-
 .../index/write/TestWALRecoveryCaching.java     |  4 +--
 .../recovery/TestPerRegionIndexWriteCache.java  |  4 +--
 .../java/org/apache/phoenix/util/TestUtil.java  |  4 +--
 26 files changed, 106 insertions(+), 115 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/it/java/org/apache/hadoop/hbase/regionserver/wal/WALReplayWithIndexWritesAndCompressedWALIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/hadoop/hbase/regionserver/wal/WALReplayWithIndexWritesAndCompressedWALIT.java b/phoenix-core/src/it/java/org/apache/hadoop/hbase/regionserver/wal/WALReplayWithIndexWritesAndCompressedWALIT.java
index e2ddd4e..49933b2 100644
--- a/phoenix-core/src/it/java/org/apache/hadoop/hbase/regionserver/wal/WALReplayWithIndexWritesAndCompressedWALIT.java
+++ b/phoenix-core/src/it/java/org/apache/hadoop/hbase/regionserver/wal/WALReplayWithIndexWritesAndCompressedWALIT.java
@@ -210,7 +210,7 @@ public class WALReplayWithIndexWritesAndCompressedWALIT {
     //make an attempted write to the primary that should also be indexed
     byte[] rowkey = Bytes.toBytes("indexed_row_key");
     Put p = new Put(rowkey);
-    p.add(family, Bytes.toBytes("qual"), Bytes.toBytes("value"));
+    p.addColumn(family, Bytes.toBytes("qual"), Bytes.toBytes("value"));
     region.put(p);
 
     // we should then see the server go down

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnProjectionOptimizationIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnProjectionOptimizationIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnProjectionOptimizationIT.java
index e4ff66f..43dc302 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnProjectionOptimizationIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnProjectionOptimizationIT.java
@@ -242,19 +242,19 @@ public class ColumnProjectionOptimizationIT extends ParallelStatsDisabledIT {
         try {
             htable = conn2.getQueryServices().getTable(htableName);
             Put put = new Put(PInteger.INSTANCE.toBytes(1));
-            put.add(cfB, c1, PInteger.INSTANCE.toBytes(1));
-            put.add(cfC, c2, PLong.INSTANCE.toBytes(2));
+            put.addColumn(cfB, c1, PInteger.INSTANCE.toBytes(1));
+            put.addColumn(cfC, c2, PLong.INSTANCE.toBytes(2));
             htable.put(put);
 
             put = new Put(PInteger.INSTANCE.toBytes(2));
-            put.add(cfC, c2, PLong.INSTANCE.toBytes(10));
-            put.add(cfC, c3, PVarchar.INSTANCE.toBytes("abcd"));
+            put.addColumn(cfC, c2, PLong.INSTANCE.toBytes(10));
+            put.addColumn(cfC, c3, PVarchar.INSTANCE.toBytes("abcd"));
             htable.put(put);
 
             put = new Put(PInteger.INSTANCE.toBytes(3));
-            put.add(cfB, c1, PInteger.INSTANCE.toBytes(3));
-            put.add(cfC, c2, PLong.INSTANCE.toBytes(10));
-            put.add(cfC, c3, PVarchar.INSTANCE.toBytes("abcd"));
+            put.addColumn(cfB, c1, PInteger.INSTANCE.toBytes(3));
+            put.addColumn(cfC, c2, PLong.INSTANCE.toBytes(10));
+            put.addColumn(cfC, c3, PVarchar.INSTANCE.toBytes("abcd"));
             htable.put(put);
 
             conn2.close();

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DynamicColumnIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DynamicColumnIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DynamicColumnIT.java
index f55d01a..04402cd 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DynamicColumnIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DynamicColumnIT.java
@@ -88,12 +88,12 @@ public class DynamicColumnIT extends ParallelStatsDisabledIT {
                 byte[] key = Bytes.toBytes("entry1");
 
                 Put put = new Put(key);
-                put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, dv, Bytes.toBytes("default"));
-                put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, first, Bytes.toBytes("first"));
-                put.add(FAMILY_NAME_A, f1v1, Bytes.toBytes("f1value1"));
-                put.add(FAMILY_NAME_A, f1v2, Bytes.toBytes("f1value2"));
-                put.add(FAMILY_NAME_B, f2v1, Bytes.toBytes("f2value1"));
-                put.add(FAMILY_NAME_B, f2v2, Bytes.toBytes("f2value2"));
+                put.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, dv, Bytes.toBytes("default"));
+                put.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, first, Bytes.toBytes("first"));
+                put.addColumn(FAMILY_NAME_A, f1v1, Bytes.toBytes("f1value1"));
+                put.addColumn(FAMILY_NAME_A, f1v2, Bytes.toBytes("f1value2"));
+                put.addColumn(FAMILY_NAME_B, f2v1, Bytes.toBytes("f2value1"));
+                put.addColumn(FAMILY_NAME_B, f2v2, Bytes.toBytes("f2value2"));
                 mutations.add(put);
 
                 hTable.batch(mutations);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DynamicFamilyIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DynamicFamilyIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DynamicFamilyIT.java
index 866a8d2..acae6ee 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DynamicFamilyIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DynamicFamilyIT.java
@@ -115,25 +115,25 @@ public class DynamicFamilyIT extends ParallelStatsDisabledIT {
             Put put;
             List<Row> mutations = new ArrayList<Row>();
             put = new Put(Bytes.toBytes("entry1"));
-            put.add(A_CF, QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, ByteUtil.EMPTY_BYTE_ARRAY);
-            put.add(A_CF, ByteUtil.concat(MAX_CLICK_COUNT_DYNCOL_PREFIX, USER_ID2_BYTES), PInteger.INSTANCE.toBytes(ENTRY1_CLICK_COUNT));
-            put.add(B_CF, ByteUtil.concat(LAST_LOGIN_TIME_DYNCOL_PREFIX, USER_ID1_BYTES), PTime.INSTANCE.toBytes(ENTRY1_USER_ID1_LOGIN_TIME));
-            put.add(B_CF, ByteUtil.concat(LAST_LOGIN_TIME_DYNCOL_PREFIX, USER_ID2_BYTES), PTime.INSTANCE.toBytes(ENTRY1_USER_ID2_LOGIN_TIME));
+            put.addColumn(A_CF, QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, ByteUtil.EMPTY_BYTE_ARRAY);
+            put.addColumn(A_CF, ByteUtil.concat(MAX_CLICK_COUNT_DYNCOL_PREFIX, USER_ID2_BYTES), PInteger.INSTANCE.toBytes(ENTRY1_CLICK_COUNT));
+            put.addColumn(B_CF, ByteUtil.concat(LAST_LOGIN_TIME_DYNCOL_PREFIX, USER_ID1_BYTES), PTime.INSTANCE.toBytes(ENTRY1_USER_ID1_LOGIN_TIME));
+            put.addColumn(B_CF, ByteUtil.concat(LAST_LOGIN_TIME_DYNCOL_PREFIX, USER_ID2_BYTES), PTime.INSTANCE.toBytes(ENTRY1_USER_ID2_LOGIN_TIME));
             mutations.add(put);
             
             put = new Put(Bytes.toBytes("entry2"));
-            put.add(A_CF, QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, ByteUtil.EMPTY_BYTE_ARRAY);
-            put.add(A_CF, ByteUtil.concat(MAX_CLICK_COUNT_DYNCOL_PREFIX, USER_ID3_BYTES), PInteger.INSTANCE.toBytes(ENTRY2_CLICK_COUNT));
-            put.add(B_CF, ByteUtil.concat(LAST_LOGIN_TIME_DYNCOL_PREFIX, USER_ID2_BYTES), PTime.INSTANCE.toBytes(ENTRY2_USER_ID2_LOGIN_TIME));
-            put.add(B_CF, ByteUtil.concat(LAST_LOGIN_TIME_DYNCOL_PREFIX, USER_ID3_BYTES), PTime.INSTANCE.toBytes(ENTRY2_USER_ID3_LOGIN_TIME));
+            put.addColumn(A_CF, QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, ByteUtil.EMPTY_BYTE_ARRAY);
+            put.addColumn(A_CF, ByteUtil.concat(MAX_CLICK_COUNT_DYNCOL_PREFIX, USER_ID3_BYTES), PInteger.INSTANCE.toBytes(ENTRY2_CLICK_COUNT));
+            put.addColumn(B_CF, ByteUtil.concat(LAST_LOGIN_TIME_DYNCOL_PREFIX, USER_ID2_BYTES), PTime.INSTANCE.toBytes(ENTRY2_USER_ID2_LOGIN_TIME));
+            put.addColumn(B_CF, ByteUtil.concat(LAST_LOGIN_TIME_DYNCOL_PREFIX, USER_ID3_BYTES), PTime.INSTANCE.toBytes(ENTRY2_USER_ID3_LOGIN_TIME));
             mutations.add(put);
             
             put = new Put(Bytes.toBytes("entry3"));
-            put.add(A_CF, QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, ByteUtil.EMPTY_BYTE_ARRAY);
-            put.add(A_CF, ByteUtil.concat(MAX_CLICK_COUNT_DYNCOL_PREFIX, USER_ID1_BYTES), PInteger.INSTANCE.toBytes(ENTRY3_CLICK_COUNT));
-            put.add(B_CF, ByteUtil.concat(LAST_LOGIN_TIME_DYNCOL_PREFIX, USER_ID1_BYTES), PTime.INSTANCE.toBytes(ENTRY3_USER_ID1_LOGIN_TIME));
-            put.add(B_CF, ByteUtil.concat(LAST_LOGIN_TIME_DYNCOL_PREFIX, USER_ID2_BYTES), PTime.INSTANCE.toBytes(ENTRY3_USER_ID2_LOGIN_TIME));
-            put.add(B_CF, ByteUtil.concat(LAST_LOGIN_TIME_DYNCOL_PREFIX, USER_ID3_BYTES), PTime.INSTANCE.toBytes(ENTRY3_USER_ID3_LOGIN_TIME));
+            put.addColumn(A_CF, QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, ByteUtil.EMPTY_BYTE_ARRAY);
+            put.addColumn(A_CF, ByteUtil.concat(MAX_CLICK_COUNT_DYNCOL_PREFIX, USER_ID1_BYTES), PInteger.INSTANCE.toBytes(ENTRY3_CLICK_COUNT));
+            put.addColumn(B_CF, ByteUtil.concat(LAST_LOGIN_TIME_DYNCOL_PREFIX, USER_ID1_BYTES), PTime.INSTANCE.toBytes(ENTRY3_USER_ID1_LOGIN_TIME));
+            put.addColumn(B_CF, ByteUtil.concat(LAST_LOGIN_TIME_DYNCOL_PREFIX, USER_ID2_BYTES), PTime.INSTANCE.toBytes(ENTRY3_USER_ID2_LOGIN_TIME));
+            put.addColumn(B_CF, ByteUtil.concat(LAST_LOGIN_TIME_DYNCOL_PREFIX, USER_ID3_BYTES), PTime.INSTANCE.toBytes(ENTRY3_USER_ID3_LOGIN_TIME));
             mutations.add(put);
 
             hTable.batch(mutations);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MappingTableDataTypeIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MappingTableDataTypeIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MappingTableDataTypeIT.java
index 52e22bf..043907b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MappingTableDataTypeIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MappingTableDataTypeIT.java
@@ -107,8 +107,8 @@ public class MappingTableDataTypeIT extends ParallelStatsDisabledIT {
     private void insertData(final byte[] tableName, HBaseAdmin admin, HTableInterface t) throws IOException,
             InterruptedException {
         Put p = new Put(Bytes.toBytes("row"));
-        p.add(Bytes.toBytes("cf1"), Bytes.toBytes("q1"), Bytes.toBytes("value1"));
-        p.add(Bytes.toBytes("cf2"), Bytes.toBytes("q2"), Bytes.toBytes("value2"));
+        p.addColumn(Bytes.toBytes("cf1"), Bytes.toBytes("q1"), Bytes.toBytes("value1"));
+        p.addColumn(Bytes.toBytes("cf2"), Bytes.toBytes("q2"), Bytes.toBytes("value2"));
         t.put(p);
         t.flushCommits();
         admin.flush(tableName);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NativeHBaseTypesIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NativeHBaseTypesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NativeHBaseTypesIT.java
index 127c25a..50563d4 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NativeHBaseTypesIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NativeHBaseTypesIT.java
@@ -96,8 +96,8 @@ public class NativeHBaseTypesIT extends ParallelStatsDisabledIT {
             
             bKey = key = ByteUtil.concat(Bytes.toBytes(20), Bytes.toBytes(200L), Bytes.toBytes("b"));
             put = new Put(key);
-            put.add(family, uintCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(5000));
-            put.add(family, ulongCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(50000L));
+            put.addColumn(family, uintCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(5000));
+            put.addColumn(family, ulongCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(50000L));
             mutations.add(put);
             // FIXME: the version of the Delete constructor without the lock args was introduced
             // in 0.94.4, thus if we try to use it here we can no longer use the 0.94.2 version
@@ -106,30 +106,30 @@ public class NativeHBaseTypesIT extends ParallelStatsDisabledIT {
             Delete del = new Delete(key, ts);
             mutations.add(del);
             put = new Put(key);
-            put.add(family, uintCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(2000));
-            put.add(family, ulongCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(20000L));
+            put.addColumn(family, uintCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(2000));
+            put.addColumn(family, ulongCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(20000L));
             mutations.add(put);
             
             key = ByteUtil.concat(Bytes.toBytes(10), Bytes.toBytes(100L), Bytes.toBytes("a"));
             put = new Put(key);
-            put.add(family, uintCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(5));
-            put.add(family, ulongCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(50L));
+            put.addColumn(family, uintCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(5));
+            put.addColumn(family, ulongCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(50L));
             mutations.add(put);
             put = new Put(key);
-            put.add(family, uintCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(10));
-            put.add(family, ulongCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(100L));
+            put.addColumn(family, uintCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(10));
+            put.addColumn(family, ulongCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(100L));
             mutations.add(put);
             
             key = ByteUtil.concat(Bytes.toBytes(30), Bytes.toBytes(300L), Bytes.toBytes("c"));
             put = new Put(key);
-            put.add(family, uintCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(3000));
-            put.add(family, ulongCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(30000L));
+            put.addColumn(family, uintCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(3000));
+            put.addColumn(family, ulongCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(30000L));
             mutations.add(put);
             
             key = ByteUtil.concat(Bytes.toBytes(40), Bytes.toBytes(400L), Bytes.toBytes("d"));
             put = new Put(key);
-            put.add(family, uintCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(4000));
-            put.add(family, ulongCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(40000L));
+            put.addColumn(family, uintCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(4000));
+            put.addColumn(family, ulongCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(40000L));
             mutations.add(put);
             
             hTable.batch(mutations);
@@ -286,9 +286,9 @@ public class NativeHBaseTypesIT extends ParallelStatsDisabledIT {
         // negative number for an unsigned type
         key = ByteUtil.concat(Bytes.toBytes(-10), Bytes.toBytes(100L), Bytes.toBytes("e"));
         put = new Put(key);
-        put.add(family, uintCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(10));
-        put.add(family, ulongCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(100L));
-        put.add(family, QueryConstants.EMPTY_COLUMN_BYTES, HConstants.LATEST_TIMESTAMP, ByteUtil.EMPTY_BYTE_ARRAY);
+        put.addColumn(family, uintCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(10));
+        put.addColumn(family, ulongCol, HConstants.LATEST_TIMESTAMP, Bytes.toBytes(100L));
+        put.addColumn(family, QueryConstants.EMPTY_COLUMN_BYTES, HConstants.LATEST_TIMESTAMP, ByteUtil.EMPTY_BYTE_ARRAY);
         mutations.add(put);
         hTable.batch(mutations);
     

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
index c7c8ebf..4f84304 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
@@ -899,8 +899,8 @@ public class QueryDatabaseMetaDataIT extends ParallelStatsDisabledIT {
                     pconn.getQueryServices()
                             .getTable(SchemaUtil.getTableNameAsBytes(schemaName, tableName));
             Put put = new Put(Bytes.toBytes("0"));
-            put.add(cfB, Bytes.toBytes("COL1"), PInteger.INSTANCE.toBytes(1));
-            put.add(cfC, Bytes.toBytes("COL2"), PLong.INSTANCE.toBytes(2));
+            put.addColumn(cfB, Bytes.toBytes("COL1"), PInteger.INSTANCE.toBytes(1));
+            put.addColumn(cfC, Bytes.toBytes("COL2"), PLong.INSTANCE.toBytes(2));
             htable.put(put);
 
             // Should be ok b/c we've marked the view with IMMUTABLE_ROWS=true

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
index b71dd7c..48a49b2 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
@@ -624,7 +624,7 @@ public class UpgradeIT extends ParallelStatsDisabledIT {
                     schemaName == null ? new byte[0] : Bytes.toBytes(schemaName),
                     Bytes.toBytes(tableName));
         Put viewColumnDefinitionPut = new Put(rowKey, HConstants.LATEST_TIMESTAMP);
-        viewColumnDefinitionPut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+        viewColumnDefinitionPut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
             PhoenixDatabaseMetaData.BASE_COLUMN_COUNT_BYTES, HConstants.LATEST_TIMESTAMP, null);
 
         try (PhoenixConnection conn =
@@ -738,7 +738,7 @@ public class UpgradeIT extends ParallelStatsDisabledIT {
                 byte[] family = PhoenixDatabaseMetaData.SYSTEM_MUTEX_FAMILY_NAME_BYTES;
                 byte[] qualifier = UPGRADE_MUTEX;
                 Put put = new Put(row);
-                put.add(family, qualifier, UPGRADE_MUTEX_UNLOCKED);
+                put.addColumn(family, qualifier, UPGRADE_MUTEX_UNLOCKED);
                 sysMutexTable.put(put);
                 sysMutexTable.flushCommits();
             }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
index 5421801..ce01e2b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
@@ -273,7 +273,7 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         HTableInterface htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes( nonTxTableName));
         List<Put>puts = Lists.newArrayList(new Put(PInteger.INSTANCE.toBytes(1)), new Put(PInteger.INSTANCE.toBytes(2)), new Put(PInteger.INSTANCE.toBytes(3)));
         for (Put put : puts) {
-            put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, ByteUtil.EMPTY_BYTE_ARRAY);
+            put.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, ByteUtil.EMPTY_BYTE_ARRAY);
         }
         htable.put(puts);
         
@@ -333,7 +333,7 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         // Reset empty column value to an empty value like it is pre-transactions
         HTableInterface htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("SYSTEM." + nonTxTableName));
         Put put = new Put(PInteger.INSTANCE.toBytes(1));
-        put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, ByteUtil.EMPTY_BYTE_ARRAY);
+        put.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, ByteUtil.EMPTY_BYTE_ARRAY);
         htable.put(put);
         
         HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 34218d5..4c72c2d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -2639,7 +2639,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                             int newPosition = ++lastOrdinalPos;
                             byte[] ptr = new byte[PInteger.INSTANCE.getByteSize()];
                             PInteger.INSTANCE.getCodec().encodeInt(newPosition, ptr, 0);
-                            viewColumnPut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                            viewColumnPut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                                     PhoenixDatabaseMetaData.ORDINAL_POSITION_BYTES, clientTimeStamp, ptr);
                             mutationsForAddingColumnsToViews.add(viewColumnPut);
                         } else {
@@ -2674,7 +2674,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                         short newKeySeq = (short)(currentKeySeq + deltaNumPkColsSoFar);
                         byte[] keySeqBytes = new byte[PSmallint.INSTANCE.getByteSize()];
                         PSmallint.INSTANCE.getCodec().encodeShort(newKeySeq, keySeqBytes, 0);
-                        viewColumnPut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                        viewColumnPut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                                 PhoenixDatabaseMetaData.KEY_SEQ_BYTES, keySeqBytes);
                         addMutationsForAddingPkColsToViewIndexes(mutationsForAddingColumnsToViews, clientTimeStamp, view,
                                 deltaNumPkColsSoFar, columnName, viewColumnPut);
@@ -2713,7 +2713,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                 }
                 byte[] viewSequencePtr = new byte[PLong.INSTANCE.getByteSize()];
                 PLong.INSTANCE.getCodec().encodeLong(view.getSequenceNumber() + 1, viewSequencePtr, 0);
-                viewHeaderRowPut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                viewHeaderRowPut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                         PhoenixDatabaseMetaData.TABLE_SEQ_NUM_BYTES, clientTimeStamp, viewSequencePtr);
                 // invalidate the view so that it is removed from the cache
                 invalidateList.add(new ImmutableBytesPtr(viewKey));
@@ -2751,21 +2751,21 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
             int oldBaseColumnCount = view.getBaseColumnCount();
             byte[] baseColumnCountPtr = new byte[PInteger.INSTANCE.getByteSize()];
             PInteger.INSTANCE.getCodec().encodeInt(oldBaseColumnCount + baseTableColumnDelta, baseColumnCountPtr, 0);
-            viewHeaderRowPut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+            viewHeaderRowPut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                     PhoenixDatabaseMetaData.BASE_COLUMN_COUNT_BYTES, clientTimeStamp, baseColumnCountPtr);
         }
         
         if (viewColumnDelta != 0) {
             byte[] columnCountPtr = new byte[PInteger.INSTANCE.getByteSize()];
             PInteger.INSTANCE.getCodec().encodeInt(numCols + viewColumnDelta, columnCountPtr, 0);
-            viewHeaderRowPut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+            viewHeaderRowPut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                     PhoenixDatabaseMetaData.COLUMN_COUNT_BYTES, clientTimeStamp, columnCountPtr);
         }
         
         if (changeSequenceNumber) {
             byte[] viewSequencePtr = new byte[PLong.INSTANCE.getByteSize()];
             PLong.INSTANCE.getCodec().encodeLong(view.getSequenceNumber() + 1, viewSequencePtr, 0);
-            viewHeaderRowPut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+            viewHeaderRowPut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                     PhoenixDatabaseMetaData.TABLE_SEQ_NUM_BYTES, clientTimeStamp, viewSequencePtr);
 
             mutationsForAddingColumnsToViews.add(viewHeaderRowPut);
@@ -2781,7 +2781,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                 Put positionUpdatePut = new Put(columnKey, clientTimeStamp);
                 byte[] ptr = new byte[PInteger.INSTANCE.getByteSize()];
                 PInteger.INSTANCE.getCodec().encodeInt(ordinalPosition, ptr, 0);
-                positionUpdatePut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                positionUpdatePut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                         PhoenixDatabaseMetaData.ORDINAL_POSITION_BYTES, clientTimeStamp, ptr);
                 mutationsForAddingColumnsToViews.add(positionUpdatePut);
                 i++;
@@ -2796,7 +2796,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
         if (!basePhysicalTable.isTransactional() && switchAttribute(basePhysicalTable, basePhysicalTable.isTransactional(), tableMetadata, TRANSACTIONAL_BYTES)) {
         	invalidateList.add(new ImmutableBytesPtr(viewKey));
         	Put put = new Put(viewKey);
-            put.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+            put.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
             		TRANSACTIONAL_BYTES, clientTimeStamp, PBoolean.INSTANCE.toBytes(true));
             mutationsForAddingColumnsToViews.add(put);
         }
@@ -3034,14 +3034,14 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                 long newSequenceNumber = index.getSequenceNumber() + 1;
                 byte[] newSequenceNumberPtr = new byte[PLong.INSTANCE.getByteSize()];
                 PLong.INSTANCE.getCodec().encodeLong(newSequenceNumber, newSequenceNumberPtr, 0);
-                indexHeaderRowMutation.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                indexHeaderRowMutation.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                         PhoenixDatabaseMetaData.TABLE_SEQ_NUM_BYTES, newSequenceNumberPtr);
                 
                 // increase the column count
                 int newColumnCount = index.getColumns().size() + deltaNumPkColsSoFar;
                 byte[] newColumnCountPtr = new byte[PInteger.INSTANCE.getByteSize()];
                 PInteger.INSTANCE.getCodec().encodeInt(newColumnCount, newColumnCountPtr, 0);
-                indexHeaderRowMutation.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                indexHeaderRowMutation.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                         PhoenixDatabaseMetaData.COLUMN_COUNT_BYTES, newColumnCountPtr);
                 
                 // add index row header key to the invalidate list to force clients to fetch the latest meta-data
@@ -3068,7 +3068,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
             int indexColumnDataType = IndexUtil.getIndexColumnDataType(true,
                     PDataType.fromTypeId(viewPkColumnDataType)).getSqlType();
             PInteger.INSTANCE.getCodec().encodeInt(indexColumnDataType, indexColumnDataTypeBytes, 0);
-            indexColumnDefinitionPut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+            indexColumnDefinitionPut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                     PhoenixDatabaseMetaData.DATA_TYPE_BYTES, indexColumnDataTypeBytes);
             
              
@@ -3078,7 +3078,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                     PhoenixDatabaseMetaData.DECIMAL_DIGITS_BYTES);
             if (decimalDigits != null && decimalDigits.size() > 0) {
                 Cell decimalDigit = decimalDigits.get(0);
-                indexColumnDefinitionPut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                indexColumnDefinitionPut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                         PhoenixDatabaseMetaData.DECIMAL_DIGITS_BYTES, decimalDigit.getValueArray());
             }
             
@@ -3088,7 +3088,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                     PhoenixDatabaseMetaData.COLUMN_SIZE_BYTES);
             if (columnSizes != null && columnSizes.size() > 0) {
                 Cell columnSize = columnSizes.get(0);
-                indexColumnDefinitionPut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                indexColumnDefinitionPut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                         PhoenixDatabaseMetaData.COLUMN_SIZE_BYTES, columnSize.getValueArray());
             }
             
@@ -3097,7 +3097,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                     PhoenixDatabaseMetaData.SORT_ORDER_BYTES);
             if (sortOrders != null && sortOrders.size() > 0) {
                 Cell sortOrder = sortOrders.get(0);
-                indexColumnDefinitionPut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                indexColumnDefinitionPut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                         PhoenixDatabaseMetaData.SORT_ORDER_BYTES, sortOrder.getValueArray());
             }
             
@@ -3106,7 +3106,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                     PhoenixDatabaseMetaData.DATA_TABLE_NAME_BYTES);
             if (dataTableNames != null && dataTableNames.size() > 0) {
                 Cell dataTableName = dataTableNames.get(0);
-                indexColumnDefinitionPut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                indexColumnDefinitionPut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                         PhoenixDatabaseMetaData.DATA_TABLE_NAME_BYTES, dataTableName.getValueArray());
             }
             
@@ -3114,12 +3114,12 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
             byte[] ordinalPositionBytes = new byte[PInteger.INSTANCE.getByteSize()];
             int ordinalPositionOfNewCol = oldNumberOfColsInIndex + deltaNumPkColsSoFar;
             PInteger.INSTANCE.getCodec().encodeInt(ordinalPositionOfNewCol, ordinalPositionBytes, 0);
-            indexColumnDefinitionPut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+            indexColumnDefinitionPut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                         PhoenixDatabaseMetaData.ORDINAL_POSITION_BYTES, ordinalPositionBytes);
             
             // New PK columns have to be nullable after the first DDL
             byte[] isNullableBytes = PInteger.INSTANCE.toBytes(ResultSetMetaData.columnNullable);
-            indexColumnDefinitionPut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+            indexColumnDefinitionPut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                         PhoenixDatabaseMetaData.NULLABLE_BYTES, isNullableBytes);
             
             // Set the key sequence for the pk column to be added
@@ -3127,7 +3127,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
             short newKeySeq = (short)(currentKeySeq + deltaNumPkColsSoFar);
             byte[] keySeqBytes = new byte[PSmallint.INSTANCE.getByteSize()];
             PSmallint.INSTANCE.getCodec().encodeShort(newKeySeq, keySeqBytes, 0);
-            indexColumnDefinitionPut.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+            indexColumnDefinitionPut.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                     PhoenixDatabaseMetaData.KEY_SEQ_BYTES, keySeqBytes);
             
             mutationsForAddingColumnsToViews.add(indexColumnDefinitionPut);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 31b512a..d202193 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -754,7 +754,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
                                 if (!timeStamps.contains(kvts)) {
                                     Put put = new Put(kv.getRowArray(), kv.getRowOffset(),
                                         kv.getRowLength());
-                                    put.add(emptyCF, QueryConstants.EMPTY_COLUMN_BYTES, kvts,
+                                    put.addColumn(emptyCF, QueryConstants.EMPTY_COLUMN_BYTES, kvts,
                                         ByteUtil.EMPTY_BYTE_ARRAY);
                                     mutations.add(put);
                                 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
index 2f41dc3..bc2523d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
@@ -1154,15 +1154,15 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> {
                 ColumnReference indexColumn = coveredColumnsMap.get(ref);
                 // If table delete was single version, then index delete should be as well
                 if (deleteType == DeleteType.SINGLE_VERSION) {
-                    delete.deleteFamilyVersion(indexColumn.getFamily(), ts);
+                    delete.addFamilyVersion(indexColumn.getFamily(), ts);
                 } else {
-                    delete.deleteFamily(indexColumn.getFamily(), ts);
+                    delete.addFamily(indexColumn.getFamily(), ts);
                 }
             }
             if (deleteType == DeleteType.SINGLE_VERSION) {
-                delete.deleteFamilyVersion(emptyCF, ts);
+                delete.addFamilyVersion(emptyCF, ts);
             } else {
-                delete.deleteFamily(emptyCF, ts);
+                delete.addFamily(emptyCF, ts);
             }
             delete.setDurability(!indexWALDisabled ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
             return delete;
@@ -1181,9 +1181,9 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> {
                     ColumnReference indexColumn = coveredColumnsMap.get(ref);
                     // If point delete for data table, then use point delete for index as well
                     if (kv.getTypeByte() == KeyValue.Type.Delete.getCode()) { 
-                        delete.deleteColumn(indexColumn.getFamily(), indexColumn.getQualifier(), ts);
+                        delete.addColumn(indexColumn.getFamily(), indexColumn.getQualifier(), ts);
                     } else {
-                        delete.deleteColumns(indexColumn.getFamily(), indexColumn.getQualifier(), ts);
+                        delete.addColumns(indexColumn.getFamily(), indexColumn.getQualifier(), ts);
                     }
                 }
             }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index f5e83f2..8866ced 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -2651,7 +2651,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                 byte[] mutexRowKey = SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA,
                         PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE);
                 Put put = new Put(mutexRowKey);
-                put.add(PhoenixDatabaseMetaData.SYSTEM_MUTEX_FAMILY_NAME_BYTES, UPGRADE_MUTEX, UPGRADE_MUTEX_UNLOCKED);
+                put.addColumn(PhoenixDatabaseMetaData.SYSTEM_MUTEX_FAMILY_NAME_BYTES, UPGRADE_MUTEX, UPGRADE_MUTEX_UNLOCKED);
                 sysMutexTable.put(put);
             }
         } catch (IOException e) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index 1a11427..082a58b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -1040,10 +1040,10 @@ public class PTableImpl implements PTable {
             newMutations();
             Delete delete = new Delete(key);
             if (families.isEmpty()) {
-                delete.deleteFamily(SchemaUtil.getEmptyColumnFamily(PTableImpl.this), ts);
+                delete.addFamily(SchemaUtil.getEmptyColumnFamily(PTableImpl.this), ts);
             } else {
                 for (PColumnFamily colFamily : families) {
-                    delete.deleteFamily(colFamily.getName().getBytes(), ts);
+                    delete.addFamily(colFamily.getName().getBytes(), ts);
                 }
             }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
index 8956862..ae077b9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
@@ -61,7 +61,6 @@ import org.apache.phoenix.util.MetaDataUtil;
 import org.apache.phoenix.util.PrefixByteDecoder;
 import org.apache.phoenix.util.SchemaUtil;
 import org.apache.phoenix.util.ServerUtil;
-import org.apache.phoenix.util.TimeKeeper;
 
 import com.google.protobuf.ServiceException;
 
@@ -189,13 +188,13 @@ public class StatisticsWriter implements Closeable {
     private void addGuidepost(ImmutableBytesPtr cfKey, List<Mutation> mutations, ImmutableBytesWritable ptr, long byteCount, long rowCount, long timeStamp) {
         byte[] prefix = StatisticsUtil.getRowKey(tableName, cfKey, ptr);
         Put put = new Put(prefix);
-        put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES,
+        put.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES,
                 timeStamp, PLong.INSTANCE.toBytes(byteCount));
-        put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
+        put.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
                 PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, timeStamp,
                 PLong.INSTANCE.toBytes(rowCount));
         // Add our empty column value so queries behave correctly
-        put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, timeStamp,
+        put.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, timeStamp,
                 ByteUtil.EMPTY_BYTE_ARRAY);
         mutations.add(put);
     }
@@ -241,7 +240,7 @@ public class StatisticsWriter implements Closeable {
         long currentTime = EnvironmentEdgeManager.currentTimeMillis();
         byte[] prefix = tableName;
         Put put = new Put(prefix);
-        put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.LAST_STATS_UPDATE_TIME_BYTES,
+        put.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.LAST_STATS_UPDATE_TIME_BYTES,
                 timeStamp, PDate.INSTANCE.toBytes(new Date(currentTime)));
         return put;
     }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 33b7383..7e280f4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -735,11 +735,11 @@ public class IndexUtil {
             HTableInterface metaTable, PIndexState newState) throws Throwable {
         // Mimic the Put that gets generated by the client on an update of the index state
         Put put = new Put(indexTableKey);
-        put.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_STATE_BYTES,
+        put.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_STATE_BYTES,
                 newState.getSerializedBytes());
-        put.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES,
+        put.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES,
                 PLong.INSTANCE.toBytes(minTimeStamp));
-        put.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.ASYNC_REBUILD_TIMESTAMP_BYTES,
+        put.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.ASYNC_REBUILD_TIMESTAMP_BYTES,
                 PLong.INSTANCE.toBytes(0));
         final List<Mutation> tableMetadata = Collections.<Mutation> singletonList(put);
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ReadWriteKeyValuesWithCodecTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ReadWriteKeyValuesWithCodecTest.java b/phoenix-core/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ReadWriteKeyValuesWithCodecTest.java
index 469dd21..8bb491d 100644
--- a/phoenix-core/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ReadWriteKeyValuesWithCodecTest.java
+++ b/phoenix-core/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ReadWriteKeyValuesWithCodecTest.java
@@ -39,12 +39,10 @@ import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.codec.Codec;
 import org.apache.hadoop.hbase.io.util.LRUDictionary;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
 import org.apache.phoenix.hbase.index.IndexTestingUtils;
 import org.apache.phoenix.hbase.index.wal.IndexedKeyValue;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.junit.experimental.categories.Category;
 
 /**
  * Simple test to read/write simple files via our custom {@link WALCellCodec} to ensure properly
@@ -93,14 +91,14 @@ public class ReadWriteKeyValuesWithCodecTest {
     // Build up a couple of edits
     List<WALEdit> edits = new ArrayList<WALEdit>();
     Put p = new Put(ROW);
-    p.add(FAMILY, null, Bytes.toBytes("v1"));
+    p.addColumn(FAMILY, null, Bytes.toBytes("v1"));
 
     WALEdit withPut = new WALEdit();
     addMutation(withPut, p, FAMILY);
     edits.add(withPut);
 
     Delete d = new Delete(ROW);
-    d.deleteColumn(FAMILY, null);
+    d.addColumn(FAMILY, null);
     WALEdit withDelete = new WALEdit();
     addMutation(withDelete, d, FAMILY);
     edits.add(withDelete);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/CoveredColumnIndexCodec.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/CoveredColumnIndexCodec.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/CoveredColumnIndexCodec.java
index a668c21..83d05f3 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/CoveredColumnIndexCodec.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/CoveredColumnIndexCodec.java
@@ -101,7 +101,7 @@ public class CoveredColumnIndexCodec extends BaseIndexCodec {
         // add each of the corresponding families to the put
         int count = 0;
         for (ColumnEntry column : columns) {
-            indexInsert.add(INDEX_ROW_COLUMN_FAMILY,
+            indexInsert.addColumn(INDEX_ROW_COLUMN_FAMILY,
                     ArrayUtils.addAll(Bytes.toBytes(count++), toIndexQualifier(column.ref)), null);
         }
     }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/LocalTableStateTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/LocalTableStateTest.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/LocalTableStateTest.java
index 82f3c3c..c7e1769 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/LocalTableStateTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/LocalTableStateTest.java
@@ -73,7 +73,7 @@ public class LocalTableStateTest {
   @Test
   public void testCorrectOrderingWithLazyLoadingColumns() throws Exception {
     Put m = new Put(row);
-    m.add(fam, qual, ts, val);
+    m.addColumn(fam, qual, ts, val);
     // setup mocks
     Configuration conf = new Configuration(false);
     RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);
@@ -132,7 +132,7 @@ public class LocalTableStateTest {
             
         };
     Put m = new Put(row);
-    m.add(fam, qual, ts, val);
+    m.addColumn(fam, qual, ts, val);
     // setup mocks
     Configuration conf = new Configuration(false);
     RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);
@@ -169,7 +169,7 @@ public class LocalTableStateTest {
             
     };
     Put m = new Put(row);
-    m.add(fam, qual, ts, val);
+    m.addColumn(fam, qual, ts, val);
     // setup mocks
     Configuration conf = new Configuration(false);
     RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);
@@ -201,7 +201,7 @@ public class LocalTableStateTest {
   @SuppressWarnings("unchecked")
   public void testCorrectRollback() throws Exception {
     Put m = new Put(row);
-    m.add(fam, qual, ts, val);
+    m.addColumn(fam, qual, ts, val);
     // setup mocks
     RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);
 
@@ -269,7 +269,7 @@ public class LocalTableStateTest {
     });
     LocalHBaseState state = new LocalTable(env);
     Put pendingUpdate = new Put(row);
-    pendingUpdate.add(fam, qual, ts, val);
+    pendingUpdate.addColumn(fam, qual, ts, val);
     LocalTableState table = new LocalTableState(state, pendingUpdate);
 
     // do the lookup for the given column

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestCoveredColumnIndexCodec.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestCoveredColumnIndexCodec.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestCoveredColumnIndexCodec.java
index 5cc6ada..d63dd6b 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestCoveredColumnIndexCodec.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/TestCoveredColumnIndexCodec.java
@@ -200,7 +200,7 @@ public class TestCoveredColumnIndexCodec {
     Delete d = new Delete(PK, 2);
     // need to set the timestamp here, as would actually happen on the server, unlike what happens
     // with puts, where the get the constructor specified timestamp for unspecified methods.
-    d.deleteFamily(FAMILY, 2);
+    d.addFamily(FAMILY, 2);
     // setup the next batch of 'current state', basically just ripping out the current state from
     // the last round
     table = new SimpleTableState(new Result(kvs));
@@ -221,12 +221,12 @@ public class TestCoveredColumnIndexCodec {
 
     // now with the delete of the columns
     d = new Delete(PK, 2);
-    d.deleteColumns(FAMILY, QUAL, 2);
+    d.addColumns(FAMILY, QUAL, 2);
     ensureNoUpdatesWhenCoveredByDelete(env, codec, kvs, d);
 
     // this delete needs to match timestamps exactly, by contract, to have any effect
     d = new Delete(PK, 1);
-    d.deleteColumn(FAMILY, QUAL, 1);
+    d.addColumn(FAMILY, QUAL, 1);
     ensureNoUpdatesWhenCoveredByDelete(env, codec, kvs, d);
   }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestIndexWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestIndexWriter.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestIndexWriter.java
index 918c411..a25f7cf 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestIndexWriter.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestIndexWriter.java
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -51,8 +50,6 @@ import org.apache.hadoop.hbase.util.VersionInfo;
 import org.apache.phoenix.hbase.index.StubAbortable;
 import org.apache.phoenix.hbase.index.TableName;
 import org.apache.phoenix.hbase.index.exception.IndexWriteException;
-import org.apache.phoenix.hbase.index.exception.SingleIndexWriteFailureException;
-import org.apache.phoenix.hbase.index.table.HTableInterfaceReference;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 import org.junit.Rule;
 import org.junit.Test;
@@ -60,9 +57,6 @@ import org.mockito.Mockito;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
-import com.google.common.collect.LinkedListMultimap;
-import com.google.common.collect.Multimap;
-
 public class TestIndexWriter {
   private static final Log LOG = LogFactory.getLog(TestIndexWriter.class);
   @Rule
@@ -115,7 +109,7 @@ public class TestIndexWriter {
 
     byte[] tableName = this.testName.getTableName();
     Put m = new Put(row);
-    m.add(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
+    m.addColumn(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
     Collection<Pair<Mutation, byte[]>> indexUpdates = Arrays.asList(new Pair<Mutation, byte[]>(m,
         tableName));
 
@@ -197,7 +191,7 @@ public class TestIndexWriter {
 
     // update a single table
     Put m = new Put(row);
-    m.add(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
+    m.addColumn(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
     final List<Pair<Mutation, byte[]>> indexUpdates = new ArrayList<Pair<Mutation, byte[]>>();
     indexUpdates.add(new Pair<Mutation, byte[]>(m, tableName));
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleIndexWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleIndexWriter.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleIndexWriter.java
index bfe1d0d..cd29e10 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleIndexWriter.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleIndexWriter.java
@@ -101,7 +101,7 @@ public class TestParalleIndexWriter {
     Mockito.when(mockRegion.getTableDesc()).thenReturn(mockTableDesc);
     ImmutableBytesPtr tableName = new ImmutableBytesPtr(this.test.getTableName());
     Put m = new Put(row);
-    m.add(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
+    m.addColumn(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
     Multimap<HTableInterfaceReference, Mutation> indexUpdates =
         ArrayListMultimap.<HTableInterfaceReference, Mutation> create();
     indexUpdates.put(new HTableInterfaceReference(tableName), m);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleWriterIndexCommitter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleWriterIndexCommitter.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleWriterIndexCommitter.java
index 6f0881b..32ae108 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleWriterIndexCommitter.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleWriterIndexCommitter.java
@@ -102,7 +102,7 @@ public class TestParalleWriterIndexCommitter {
 
     ImmutableBytesPtr tableName = new ImmutableBytesPtr(this.test.getTableName());
     Put m = new Put(row);
-    m.add(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
+    m.addColumn(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
     Multimap<HTableInterfaceReference, Mutation> indexUpdates =
         ArrayListMultimap.<HTableInterfaceReference, Mutation> create();
     indexUpdates.put(new HTableInterfaceReference(tableName), m);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestWALRecoveryCaching.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestWALRecoveryCaching.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestWALRecoveryCaching.java
index d3af9cd..b1e87e5 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestWALRecoveryCaching.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestWALRecoveryCaching.java
@@ -192,7 +192,7 @@ public class TestWALRecoveryCaching {
 
     // load some data into the table
     Put p = new Put(Bytes.toBytes("row"));
-    p.add(family, qual, Bytes.toBytes("value"));
+    p.addColumn(family, qual, Bytes.toBytes("value"));
     HTable primary = new HTable(conf, testTable.getTableName());
     primary.put(p);
     primary.flushCommits();
@@ -235,7 +235,7 @@ public class TestWALRecoveryCaching {
     // make a second put that (1), isn't indexed, so we can be sure of the index state and (2)
     // ensures that our table is back up
     Put p2 = new Put(p.getRow());
-    p2.add(nonIndexedFamily, Bytes.toBytes("Not indexed"), Bytes.toBytes("non-indexed value"));
+    p2.addColumn(nonIndexedFamily, Bytes.toBytes("Not indexed"), Bytes.toBytes("non-indexed value"));
     primary.put(p2);
     primary.flushCommits();
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
index 819c7f6..7253165 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
@@ -61,8 +61,8 @@ public class TestPerRegionIndexWriteCache {
   Put p = new Put(row);
   Put p2 = new Put(Bytes.toBytes("other row"));
   {
-    p.add(family, qual, val);
-    p2.add(family, qual, val);
+    p.addColumn(family, qual, val);
+    p2.addColumn(family, qual, val);
   }
 
   HRegion r1; // FIXME: Uses private type

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc6cf43a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index 277e257..341fbec 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -786,10 +786,10 @@ public class TestUtil {
             byte[] markerRowKey = Bytes.toBytes("TO_DELETE");
            
             Put put = new Put(markerRowKey);
-            put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES);
+            put.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES);
             htable.put(put);
             Delete delete = new Delete(markerRowKey);
-            delete.deleteColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES);
+            delete.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES);
             htable.delete(delete);
             htable.close();
             if (table.isTransactional()) {


[15/50] [abbrv] phoenix git commit: PHOENIX-4605 Support running multiple transaction providers (ammendment)

Posted by td...@apache.org.
PHOENIX-4605 Support running multiple transaction providers (ammendment)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a0b0c1e1
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a0b0c1e1
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a0b0c1e1

Branch: refs/heads/system-catalog
Commit: a0b0c1e1e0597024e7f94c6d62e238ce1f2778ef
Parents: de83b8d
Author: James Taylor <jt...@salesforce.com>
Authored: Thu Apr 12 18:11:19 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Thu Apr 12 18:12:58 2018 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/phoenix/schema/MetaDataClient.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/a0b0c1e1/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 3c7a837..1fb668e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -3276,7 +3276,7 @@ public class MetaDataClient {
                 // If changing isImmutableRows to true or it's not being changed and is already true
                 boolean willBeImmutableRows = Boolean.TRUE.equals(metaPropertiesEvaluated.getIsImmutableRows()) || (metaPropertiesEvaluated.getIsImmutableRows() == null && table.isImmutableRows());
                 boolean willBeTxnl = metaProperties.getNonTxToTx();
-                Long timeStamp = TransactionUtil.getTableTimestamp(connection, table.isTransactional() || willBeTxnl, metaPropertiesEvaluated.getTransactionProvider());
+                Long timeStamp = TransactionUtil.getTableTimestamp(connection, table.isTransactional() || willBeTxnl, table.isTransactional() ? table.getTransactionProvider() : metaPropertiesEvaluated.getTransactionProvider());
                 int numPkColumnsAdded = 0;
                 List<PColumn> columns = Lists.newArrayListWithExpectedSize(numCols);
                 Set<String> colFamiliesForPColumnsToBeAdded = new LinkedHashSet<>();


[31/50] [abbrv] phoenix git commit: Add test UpgradeIT.testMoveParentChildLinks()

Posted by td...@apache.org.
Add test UpgradeIT.testMoveParentChildLinks()


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/f30994f6
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/f30994f6
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/f30994f6

Branch: refs/heads/system-catalog
Commit: f30994f60f61c00135136b3aabaabc1193ea2cdf
Parents: 41f8142
Author: Thomas D'Silva <td...@apache.org>
Authored: Mon Apr 23 16:31:41 2018 -0700
Committer: Thomas D'Silva <td...@apache.org>
Committed: Mon Apr 23 16:31:41 2018 -0700

----------------------------------------------------------------------
 .../java/org/apache/phoenix/end2end/UpgradeIT.java  | 12 ++++++------
 .../phoenix/query/ConnectionQueryServicesImpl.java  | 16 ++++++++--------
 .../java/org/apache/phoenix/util/UpgradeUtil.java   |  4 ++--
 3 files changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/f30994f6/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
index ac0cbd3..eb362fb 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
@@ -20,7 +20,6 @@ package org.apache.phoenix.end2end;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static org.apache.phoenix.query.ConnectionQueryServicesImpl.UPGRADE_MUTEX;
 import static org.apache.phoenix.query.ConnectionQueryServicesImpl.UPGRADE_MUTEX_UNLOCKED;
-import static org.apache.phoenix.util.UpgradeUtil.SELECT_BASE_COLUMN_COUNT_FROM_HEADER_ROW;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
@@ -42,11 +41,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.curator.shaded.com.google.common.collect.Sets;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.coprocessor.MetaDataProtocol;
@@ -549,7 +546,7 @@ public class UpgradeIT extends ParallelStatsDisabledIT {
     }
     
     @Test
-    public void testAddParentChildLinks() throws Exception {
+    public void testMoveParentChildLinks() throws Exception {
         String schema = "S_" + generateUniqueName();
         String table1 = "T_" + generateUniqueName();
         String table2 = "T_" + generateUniqueName();
@@ -589,13 +586,16 @@ public class UpgradeIT extends ParallelStatsDisabledIT {
             Set<String> expectedChildLinkSet = getChildLinks(conn);
 
             // delete all the child links
-            conn.createStatement().execute("DELETE FROM SYSTEM.CATALOG WHERE LINK_TYPE = "
+            conn.createStatement().execute("DELETE FROM SYSTEM.CHILD_LINK WHERE LINK_TYPE = "
                     + LinkType.CHILD_TABLE.getSerializedValue());
 
             // re-create them by running the upgrade code
             PhoenixConnection phxMetaConn = metaConn.unwrap(PhoenixConnection.class);
             phxMetaConn.setRunningUpgrade(true);
+            // create the parent-> child links in SYSTEM.CATALOG
             UpgradeUtil.addParentToChildLinks(phxMetaConn);
+            // move the parent->child links to SYSTEM.CHILD_LINK
+            UpgradeUtil.moveChildLinks(phxMetaConn);
             Set<String> actualChildLinkSet = getChildLinks(conn);
 
             assertEquals("Unexpected child links", expectedChildLinkSet, actualChildLinkSet);
@@ -605,7 +605,7 @@ public class UpgradeIT extends ParallelStatsDisabledIT {
     private Set<String> getChildLinks(Connection conn) throws SQLException {
         ResultSet rs =
                 conn.createStatement().executeQuery(
-                    "SELECT TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY FROM SYSTEM.CATALOG WHERE LINK_TYPE = "
+                    "SELECT TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY FROM SYSTEM.CHILD_LINK WHERE LINK_TYPE = "
                             + LinkType.CHILD_TABLE.getSerializedValue());
         Set<String> childLinkSet = Sets.newHashSet();
         while (rs.next()) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/f30994f6/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 00e1d62..8bd06bc 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -57,7 +57,11 @@ import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RENEW_LEASE_
 import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RENEW_LEASE_THREAD_POOL_SIZE;
 import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RENEW_LEASE_THRESHOLD_MILLISECONDS;
 import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS;
-import static org.apache.phoenix.util.UpgradeUtil.*;
+import static org.apache.phoenix.util.UpgradeUtil.addParentToChildLinks;
+import static org.apache.phoenix.util.UpgradeUtil.addViewIndexToParentLinks;
+import static org.apache.phoenix.util.UpgradeUtil.getSysCatalogSnapshotName;
+import static org.apache.phoenix.util.UpgradeUtil.moveChildLinks;
+import static org.apache.phoenix.util.UpgradeUtil.upgradeTo4_5_0;
 
 import java.io.IOException;
 import java.lang.ref.WeakReference;
@@ -2865,14 +2869,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                                 + PBoolean.INSTANCE.getSqlTypeName());
                     addParentToChildLinks(metaConnection);
                 }
-                if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_13_0) {
-                    metaConnection = addColumnsIfNotExists(
-                        metaConnection,
-                        PhoenixDatabaseMetaData.SYSTEM_CATALOG,
-                        MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0,
-                        PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION + " "
-                                + PBoolean.INSTANCE.getSqlTypeName());
+                // TODO set the version for which the following upgrade code runs correct
+                if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0) {
                     addViewIndexToParentLinks(metaConnection);
+                    moveChildLinks(metaConnection);
                 }
             }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/f30994f6/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
index fe6470b..e1c82b2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
@@ -1159,8 +1159,8 @@ public class UpgradeUtil {
             metaConnection = new PhoenixConnection(oldMetaConnection, HConstants.LATEST_TIMESTAMP);
             logger.info("Upgrading metadata to add parent to child links for views");
             metaConnection.commit();
-            String createChildLink = "UPSERT INTO SYSTEM.CHILD_LINK(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, LINK_TYPE)" +
-                                        "SELECT TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, LINK_TYPE" + 
+            String createChildLink = "UPSERT INTO SYSTEM.CHILD_LINK(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, LINK_TYPE) " +
+                                        "SELECT TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, LINK_TYPE " + 
                                         "FROM SYSTEM.CATALOG " + 
                                         "WHERE LINK_TYPE = 4";
             metaConnection.createStatement().execute(createChildLink);


[38/50] [abbrv] phoenix git commit: PHOENIX-4694 Prevent locking of parent table when dropping view to reduce contention

Posted by td...@apache.org.
PHOENIX-4694 Prevent locking of parent table when dropping view to reduce contention


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/9168f669
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/9168f669
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/9168f669

Branch: refs/heads/system-catalog
Commit: 9168f6698d6afdf12a825bd6a4e05ccb2a85b9d0
Parents: 5c637e6
Author: James Taylor <jt...@salesforce.com>
Authored: Thu Apr 19 15:09:43 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Thu Apr 26 09:55:28 2018 -0700

----------------------------------------------------------------------
 .../java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/9168f669/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index ddd3ffe..29eee7e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -2007,8 +2007,9 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
             }
             List<byte[]> tableNamesToDelete = Lists.newArrayList();
             List<SharedTableState> sharedTablesToDelete = Lists.newArrayList();
+            // No need to lock parent table for views
             byte[] parentTableName = MetaDataUtil.getParentTableName(tableMetadata);
-            byte[] lockTableName = parentTableName == null ? tableName : parentTableName;
+            byte[] lockTableName = parentTableName == null || tableType.equals(PTableType.VIEW.getSerializedValue()) ? tableName : parentTableName;
             byte[] lockKey = SchemaUtil.getTableKey(tenantIdBytes, schemaName, lockTableName);
             byte[] key =
                     parentTableName == null ? lockKey : SchemaUtil.getTableKey(tenantIdBytes,


[32/50] [abbrv] phoenix git commit: PHOENIX-4645 PhoenixStorageHandler doesn't handle correctly data/timestamp in push down predicate when engine is tez.(Sergey Soldatov)

Posted by td...@apache.org.
PHOENIX-4645 PhoenixStorageHandler doesn't handle correctly data/timestamp in push down predicate when engine is tez.(Sergey Soldatov)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8f1cef82
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8f1cef82
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8f1cef82

Branch: refs/heads/system-catalog
Commit: 8f1cef824b086c7c697688767e0460c18fa554d6
Parents: 78636a3
Author: Rajeshbabu Chintaguntla <ra...@apache.org>
Authored: Tue Apr 24 11:24:43 2018 +0530
Committer: Rajeshbabu Chintaguntla <ra...@apache.org>
Committed: Tue Apr 24 11:24:43 2018 +0530

----------------------------------------------------------------------
 .../apache/phoenix/hive/HivePhoenixStoreIT.java | 27 ++++++++++++++++++++
 .../PhoenixStorageHandlerConstants.java         |  8 +++---
 .../phoenix/hive/query/PhoenixQueryBuilder.java |  4 +--
 3 files changed, 33 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/8f1cef82/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java
index 1828818..66f99ad 100644
--- a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java
+++ b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java
@@ -309,6 +309,33 @@ public class HivePhoenixStoreIT  extends BaseHivePhoenixStoreIT {
             assert (rs.getString(2).equalsIgnoreCase("part1"));
             assert (rs.getDouble(3) == 200);
         }
+    }
+
+    @Test
+    public void testTimestampPredicate() throws Exception {
+        String testName = "testTimeStampPredicate";
+        hbaseTestUtil.getTestFileSystem().createNewFile(new Path(hiveLogDir, testName + ".out"));
+        createFile("10\t2013-01-02 01:01:01.123456\n", new Path(hiveOutputDir, testName + ".out").toString());
+        createFile(StringUtil.EMPTY_STRING, new Path(hiveLogDir, testName + ".out").toString());
 
+        StringBuilder sb = new StringBuilder();
+        sb.append("CREATE TABLE timeStampTable(ID int,ts TIMESTAMP)" + HiveTestUtil.CRLF +
+                " STORED BY  \"org.apache.phoenix.hive.PhoenixStorageHandler\"" + HiveTestUtil
+                .CRLF +
+                " TBLPROPERTIES(" + HiveTestUtil.CRLF +
+                "   'phoenix.hbase.table.name'='TIMESTAMPTABLE'," + HiveTestUtil.CRLF +
+                "   'phoenix.zookeeper.znode.parent'='/hbase'," + HiveTestUtil.CRLF +
+                "   'phoenix.zookeeper.quorum'='localhost'," + HiveTestUtil.CRLF +
+                "   'phoenix.zookeeper.client.port'='" +
+                hbaseTestUtil.getZkCluster().getClientPort() + "'," + HiveTestUtil.CRLF +
+                "   'phoenix.column.mapping' = 'id:ID, ts:TS'," + HiveTestUtil.CRLF +
+                "   'phoenix.rowkeys'='id');" + HiveTestUtil.CRLF);
+        sb.append("INSERT INTO TABLE timeStampTable VALUES (10, \"2013-01-02 01:01:01.123456\");" + HiveTestUtil.CRLF);
+        sb.append("SELECT * from timeStampTable WHERE ts between '2013-01-02 01:01:01.123455' and " +
+                " '2013-01-02 12:01:02.123457789' AND id = 10;" + HiveTestUtil.CRLF);
+
+        String fullPath = new Path(hbaseTestUtil.getDataTestDir(), testName).toString();
+        createFile(sb.toString(), fullPath);
+        runTest(testName, fullPath);
     }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/8f1cef82/phoenix-hive/src/main/java/org/apache/phoenix/hive/constants/PhoenixStorageHandlerConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/constants/PhoenixStorageHandlerConstants.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/constants/PhoenixStorageHandlerConstants.java
index 07c374e..e3c7d84 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/constants/PhoenixStorageHandlerConstants.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/constants/PhoenixStorageHandlerConstants.java
@@ -90,13 +90,13 @@ public class PhoenixStorageHandlerConstants {
     public static final String PATERN_MARKER = "$targetPattern$";
     public static final String DATE_PATTERN = "'?\\d{4}-\\d{2}-\\d{2}'?";
     public static final String TIMESTAMP_PATTERN = "'?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\" +
-            ".?\\d{0,3}'?";
-    public static final String COMMON_OPERATOR_PATTERN = "(\\(?" + COLUMNE_MARKER + "\\)?\\s*" +
+            ".?\\d{0,9}'?";
+    public static final String COMMON_OPERATOR_PATTERN = "(\\(?\"?" + COLUMNE_MARKER + "\"?\\)?\\s*" +
             "(=|>|<|<=|>=)\\s*(" + PATERN_MARKER + "))";
-    public static final String BETWEEN_OPERATOR_PATTERN = "(\\(?" + COLUMNE_MARKER + "\\)?\\s*(" +
+    public static final String BETWEEN_OPERATOR_PATTERN = "(\\(?\"?" + COLUMNE_MARKER + "\"?\\)?\\s*(" +
             "(?i)not)?\\s*(?i)between\\s*(" + PATERN_MARKER + ")\\s*(?i)and\\s*(" + PATERN_MARKER
             + "))";
-    public static final String IN_OPERATOR_PATTERN = "(\\(?" + COLUMNE_MARKER + "\\)?\\s*((?i)" +
+    public static final String IN_OPERATOR_PATTERN = "(\\(?\"?" + COLUMNE_MARKER + "\"?\\)?\\s*((?i)" +
             "not)?\\s*(?i)in\\s*\\((" + PATERN_MARKER + ",?\\s*)+\\))";
 
     public static final String FUNCTION_VALUE_MARKER = "$value$";

http://git-wip-us.apache.org/repos/asf/phoenix/blob/8f1cef82/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
index eb336dc..ab409ad 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
@@ -255,10 +255,10 @@ public class PhoenixQueryBuilder {
 
                 if (PhoenixStorageHandlerConstants.DATE_TYPE.equals(
                         columnTypeMap.get(columnName).getTypeName())) {
-                    whereClause = applyDateFunctionUsingRegex(whereClause, columnName);
+                    whereClause = applyDateFunctionUsingRegex(whereClause, column);
                 } else if (PhoenixStorageHandlerConstants.TIMESTAMP_TYPE.equals(
                         columnTypeMap.get(columnName).getTypeName())) {
-                    whereClause = applyTimestampFunctionUsingRegex(whereClause, columnName);
+                    whereClause = applyTimestampFunctionUsingRegex(whereClause, column);
                 }
             }
         }


[21/50] [abbrv] phoenix git commit: PHOENIX-4575 Phoenix metadata KEEP_DELETED_CELLS and VERSIONS should be property driven

Posted by td...@apache.org.
PHOENIX-4575 Phoenix metadata KEEP_DELETED_CELLS and VERSIONS should be property driven


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/430e8020
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/430e8020
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/430e8020

Branch: refs/heads/system-catalog
Commit: 430e8020ce5b900d9d9a3533fc5d647926704b7f
Parents: a5770a5
Author: James Taylor <jt...@salesforce.com>
Authored: Fri Apr 13 14:30:00 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Apr 13 14:30:00 2018 -0700

----------------------------------------------------------------------
 .../phoenix/coprocessor/MetaDataProtocol.java   |  4 --
 .../query/ConnectionQueryServicesImpl.java      | 41 ++++++++++++++------
 .../query/ConnectionlessQueryServicesImpl.java  | 35 +++++++++++++++--
 .../apache/phoenix/query/QueryConstants.java    | 14 +++----
 .../org/apache/phoenix/query/QueryServices.java |  5 ++-
 .../phoenix/query/QueryServicesOptions.java     |  3 +-
 .../org/apache/phoenix/util/UpgradeUtil.java    |  4 +-
 7 files changed, 74 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/430e8020/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index 26f8198..36d6f0d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -72,10 +72,6 @@ public abstract class MetaDataProtocol extends MetaDataService {
     public static final long MIN_SYSTEM_TABLE_MIGRATION_TIMESTAMP = 0;
     public static final String MIGRATION_IN_PROGRESS = "MigrationInProgress";
 
-    public static final int DEFAULT_MAX_META_DATA_VERSIONS = 1000;
-    public static final boolean DEFAULT_META_DATA_KEEP_DELETED_CELLS = true;
-    public static final int DEFAULT_MAX_STAT_DATA_VERSIONS = 1;
-    public static final boolean DEFAULT_STATS_KEEP_DELETED_CELLS = false;
     public static final int DEFAULT_LOG_VERSIONS = 10;
     public static final int DEFAULT_LOG_TTL = 7 * 24 * 60 * 60; // 7 days 
     

http://git-wip-us.apache.org/repos/asf/phoenix/blob/430e8020/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 4c1902d..5985705 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -720,10 +720,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     private HColumnDescriptor generateColumnFamilyDescriptor(Pair<byte[],Map<String,Object>> family, PTableType tableType) throws SQLException {
         HColumnDescriptor columnDesc = new HColumnDescriptor(family.getFirst());
         if (tableType != PTableType.VIEW) {
-            if(props.get(QueryServices.DEFAULT_KEEP_DELETED_CELLS_ATTRIB) != null){
-                columnDesc.setKeepDeletedCells(props.getBoolean(
-                        QueryServices.DEFAULT_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_KEEP_DELETED_CELLS));
-            }
             columnDesc.setDataBlockEncoding(SchemaUtil.DEFAULT_DATA_BLOCK_ENCODING);
             columnDesc.setBloomFilterType(BloomType.NONE);
             for (Entry<String,Object> entry : family.getSecond().entrySet()) {
@@ -2459,8 +2455,29 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     }
 
     // Available for testing
-    protected String getSystemCatalogDML() {
-        return QueryConstants.CREATE_TABLE_METADATA;
+    protected String getSystemCatalogTableDDL() {
+        return setSystemDDLProperties(QueryConstants.CREATE_TABLE_METADATA);
+    }
+
+    // Available for testing
+    protected String getFunctionTableDDL() {
+        return setSystemDDLProperties(QueryConstants.CREATE_FUNCTION_METADATA);
+    }
+
+    // Available for testing
+    protected String getLogTableDDL() {
+        return setSystemLogDDLProperties(QueryConstants.CREATE_LOG_METADATA);
+    }
+
+    private String setSystemDDLProperties(String ddl) {
+        return String.format(ddl,
+          props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS),
+          props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS));
+    }
+
+    private String setSystemLogDDLProperties(String ddl) {
+        return String.format(ddl,
+          props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS));
     }
 
     @Override
@@ -2503,7 +2520,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                                          scnProps, newEmptyMetaData())) {
                                 try {
                                     metaConnection.setRunningUpgrade(true);
-                                    metaConnection.createStatement().executeUpdate(getSystemCatalogDML());
+                                    metaConnection.createStatement().executeUpdate(getSystemCatalogTableDDL());
                                 } catch (NewerTableAlreadyExistsException ignore) {
                                     // Ignore, as this will happen if the SYSTEM.CATALOG already exists at this fixed
                                     // timestamp. A TableAlreadyExistsException is not thrown, since the table only exists
@@ -2665,10 +2682,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
             metaConnection.createStatement().execute(QueryConstants.CREATE_STATS_TABLE_METADATA);
         } catch (TableAlreadyExistsException ignore) {}
         try {
-            metaConnection.createStatement().execute(QueryConstants.CREATE_FUNCTION_METADATA);
+            metaConnection.createStatement().execute(getFunctionTableDDL());
         } catch (TableAlreadyExistsException ignore) {}
         try {
-            metaConnection.createStatement().execute(QueryConstants.CREATE_LOG_METADATA);
+            metaConnection.createStatement().execute(getLogTableDDL());
         } catch (TableAlreadyExistsException ignore) {}
         // Catch the IOException to log the error message and then bubble it up for the client to retry.
         try {
@@ -2971,7 +2988,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                 createSysMutexTableIfNotExists(admin);
             }
             try {
-                metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_TABLE_METADATA);
+                metaConnection.createStatement().executeUpdate(getSystemCatalogTableDDL());
             } catch (NewerTableAlreadyExistsException ignore) {
                 // Ignore, as this will happen if the SYSTEM.CATALOG already exists at this fixed
                 // timestamp. A TableAlreadyExistsException is not thrown, since the table only exists
@@ -3093,10 +3110,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                 }
             }
             try {
-                metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_FUNCTION_METADATA);
+                metaConnection.createStatement().executeUpdate(getFunctionTableDDL());
             } catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {}
             try {
-                metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_LOG_METADATA);
+                metaConnection.createStatement().executeUpdate(getLogTableDDL());
             } catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {}
 
             // In case namespace mapping is enabled and system table to system namespace mapping is also enabled,

http://git-wip-us.apache.org/repos/asf/phoenix/blob/430e8020/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 1c56c31..aba9600 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -84,11 +84,13 @@ import org.apache.phoenix.schema.stats.GuidePostsInfo;
 import org.apache.phoenix.schema.stats.GuidePostsKey;
 import org.apache.phoenix.transaction.PhoenixTransactionClient;
 import org.apache.phoenix.transaction.TransactionFactory.Provider;
+import org.apache.phoenix.util.ConfigUtil;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.JDBCUtil;
 import org.apache.phoenix.util.MetaDataUtil;
 import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
 import org.apache.phoenix.util.SchemaUtil;
 import org.apache.phoenix.util.SequenceUtil;
 
@@ -106,6 +108,7 @@ import com.google.common.collect.Maps;
 public class ConnectionlessQueryServicesImpl extends DelegateQueryServices implements ConnectionQueryServices  {
     private static ServerName SERVER_NAME = ServerName.parseServerName(HConstants.LOCALHOST + Addressing.HOSTNAME_PORT_SEPARATOR + HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
     
+    private final ReadOnlyProps props;
     private PMetaData metaData;
     private final Map<SequenceKey, SequenceInfo> sequenceMap = Maps.newHashMap();
     private final String userName;
@@ -143,12 +146,38 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
         // on the server side during testing.
         this.config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration(config);
         this.guidePostsCache = new GuidePostsCache(this, config);
+        // set replication required parameter
+        ConfigUtil.setReplicationConfigIfAbsent(this.config);
+        this.props = new ReadOnlyProps(this.config.iterator());
     }
 
     private PMetaData newEmptyMetaData() {
         return new PMetaDataImpl(INITIAL_META_DATA_TABLE_CAPACITY, getProps());
     }
 
+    protected String getSystemCatalogTableDDL() {
+        return setSystemDDLProperties(QueryConstants.CREATE_TABLE_METADATA);
+    }
+
+    protected String getFunctionTableDDL() {
+        return setSystemDDLProperties(QueryConstants.CREATE_FUNCTION_METADATA);
+    }
+
+    protected String getLogTableDDL() {
+        return setSystemLogDDLProperties(QueryConstants.CREATE_LOG_METADATA);
+    }
+
+    private String setSystemDDLProperties(String ddl) {
+        return String.format(ddl,
+          props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS),
+          props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS));
+    }
+
+    private String setSystemLogDDLProperties(String ddl) {
+        return String.format(ddl,
+          props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS));
+    }
+
     @Override
     public ConnectionQueryServices getChildQueryServices(ImmutableBytesWritable childId) {
         return this; // Just reuse the same query services
@@ -307,7 +336,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
                 metaConnection = new PhoenixConnection(this, globalUrl, scnProps, newEmptyMetaData());
                 metaConnection.setRunningUpgrade(true);
                 try {
-                    metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_TABLE_METADATA);
+                    metaConnection.createStatement().executeUpdate(getSystemCatalogTableDDL());
                 } catch (TableAlreadyExistsException ignore) {
                     // Ignore, as this will happen if the SYSTEM.TABLE already exists at this fixed timestamp.
                     // A TableAlreadyExistsException is not thrown, since the table only exists *after* this fixed timestamp.
@@ -330,11 +359,11 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
                 }
                 
                 try {
-                   metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_FUNCTION_METADATA);
+                    metaConnection.createStatement().executeUpdate(getFunctionTableDDL());
                 } catch (NewerTableAlreadyExistsException ignore) {
                 }
                 try {
-                    metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_LOG_METADATA);
+                    metaConnection.createStatement().executeUpdate(getLogTableDDL());
                 } catch (NewerTableAlreadyExistsException ignore) {}
             } catch (SQLException e) {
                 sqlE = e;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/430e8020/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index d181fc8..22fa2f4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -343,8 +343,8 @@ public interface QueryConstants {
             TRANSACTION_PROVIDER + " TINYINT, " +
             "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ","
             + TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" +
-            HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
-            HColumnDescriptor.KEEP_DELETED_CELLS + "="  + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n" +
+            HConstants.VERSIONS + "=%s,\n" +
+            HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n" +
             // Install split policy to prevent a tenant's metadata from being split across regions.
             HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + 
             PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
@@ -361,8 +361,6 @@ public interface QueryConstants {
             "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY ("
             + PHYSICAL_NAME + ","
             + COLUMN_FAMILY + ","+ GUIDE_POST_KEY+"))\n" +
-            HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_STAT_DATA_VERSIONS + ",\n" +
-            HColumnDescriptor.KEEP_DELETED_CELLS + "="  + MetaDataProtocol.DEFAULT_STATS_KEEP_DELETED_CELLS + ",\n" +
             // Install split policy to prevent a physical table's stats from being split across regions.
             HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + 
             PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
@@ -382,8 +380,6 @@ public interface QueryConstants {
             CYCLE_FLAG + " BOOLEAN, \n" +
             LIMIT_REACHED_FLAG + " BOOLEAN \n" +
             " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + SEQUENCE_SCHEMA + "," + SEQUENCE_NAME + "))\n" +
-            HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
-            HColumnDescriptor.KEEP_DELETED_CELLS + "="  + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n" +
             PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
     public static final String CREATE_SYSTEM_SCHEMA = "CREATE SCHEMA " + SYSTEM_CATALOG_SCHEMA;
     public static final String UPGRADE_TABLE_SNAPSHOT_PREFIX = "_UPGRADING_TABLE_";
@@ -407,8 +403,8 @@ public interface QueryConstants {
             MIN_VALUE + " VARCHAR, \n" +
             MAX_VALUE + " VARCHAR, \n" +
             " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ", " + FUNCTION_NAME + ", " + TYPE + ", " + ARG_POSITION + "))\n" +
-            HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
-            HColumnDescriptor.KEEP_DELETED_CELLS + "="  + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n"+
+            HConstants.VERSIONS + "=%s,\n" +
+            HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n"+
             // Install split policy to prevent a tenant's metadata from being split across regions.
             HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + 
             PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
@@ -434,7 +430,7 @@ public interface QueryConstants {
             SCAN_METRICS_JSON + " VARCHAR, \n" +
             " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (QUERY_ID))\n" +
             HConstants.VERSIONS + "= " + MetaDataProtocol.DEFAULT_LOG_VERSIONS + ",\n" +
-            HColumnDescriptor.KEEP_DELETED_CELLS + "="  + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n"+
+            HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n"+
             // Install split policy to prevent a tenant's metadata from being split across regions.
             HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + 
             PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE+ ",\n" + 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/430e8020/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 29d18d9..8cc156c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -204,7 +204,6 @@ public interface QueryServices extends SQLCloseable {
     public static final String ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = "hbase.online.schema.update.enable";
     public static final String NUM_RETRIES_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.retries";
     public static final String DELAY_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.delay";
-    public static final String DEFAULT_KEEP_DELETED_CELLS_ATTRIB = "phoenix.table.default.keep.deleted.cells";
     public static final String DEFAULT_STORE_NULLS_ATTRIB = "phoenix.table.default.store.nulls";
     public static final String DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB = "phoenix.table.istransactional.default";
     public static final String DEFAULT_TRANSACTION_PROVIDER_ATTRIB = "phoenix.table.transaction.provider.default";
@@ -252,6 +251,10 @@ public interface QueryServices extends SQLCloseable {
     public static final String QUERY_SERVER_REMOTEUSEREXTRACTOR_PARAM = "phoenix.queryserver.remoteUserExtractor.param";
     public static final String QUERY_SERVER_DISABLE_KERBEROS_LOGIN = "phoenix.queryserver.disable.kerberos.login";
 
+    // metadata configs
+    public static final String DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB = "phoenix.system.default.keep.deleted.cells";
+    public static final String DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB = "phoenix.system.default.max.versions";
+
     public static final String RENEW_LEASE_ENABLED = "phoenix.scanner.lease.renew.enabled";
     public static final String RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS = "phoenix.scanner.lease.renew.interval";
     public static final String RENEW_LEASE_THRESHOLD_MILLISECONDS = "phoenix.scanner.lease.threshold";

http://git-wip-us.apache.org/repos/asf/phoenix/blob/430e8020/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 70ac11b..23aed7c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -213,6 +213,8 @@ public class QueryServicesOptions {
     public static final boolean DEFAULT_ALLOW_LOCAL_INDEX = true;
     public static final int DEFAULT_INDEX_HANDLER_COUNT = 30;
     public static final int DEFAULT_METADATA_HANDLER_COUNT = 30;
+    public static final int DEFAULT_SYSTEM_MAX_VERSIONS = 1;
+    public static final boolean DEFAULT_SYSTEM_KEEP_DELETED_CELLS = false;
 
     // Retries when doing server side writes to SYSTEM.CATALOG
     // 20 retries with 100 pause = 230 seconds total retry time
@@ -257,7 +259,6 @@ public class QueryServicesOptions {
     public static final boolean DEFAULT_ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = true;
     public static final int DEFAULT_RETRIES_FOR_SCHEMA_UPDATE_CHECK = 10;
     public static final long DEFAULT_DELAY_FOR_SCHEMA_UPDATE_CHECK = 5 * 1000; // 5 seconds.
-    public static final boolean DEFAULT_KEEP_DELETED_CELLS = false;
     public static final boolean DEFAULT_STORE_NULLS = false;
 
     // TODO Change this to true as part of PHOENIX-1543

http://git-wip-us.apache.org/repos/asf/phoenix/blob/430e8020/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
index 741bdff..63f2183 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
@@ -223,7 +223,7 @@ public class UpgradeUtil {
 
         Scan scan = new Scan();
         scan.setRaw(true);
-        scan.setMaxVersions(MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS);
+        scan.setMaxVersions();
         ResultScanner scanner = null;
         HTableInterface source = null;
         HTableInterface target = null;
@@ -698,7 +698,7 @@ public class UpgradeUtil {
                 boolean success = false;
                 Scan scan = new Scan();
                 scan.setRaw(true);
-                scan.setMaxVersions(MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS);
+                scan.setMaxVersions();
                 HTableInterface seqTable = conn.getQueryServices().getTable(PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME_BYTES);
                 try {
                     boolean committed = false;


[49/50] [abbrv] phoenix git commit: Merge remote-tracking branch 'upstream/master' into PHOENIX-3534

Posted by td...@apache.org.
Merge remote-tracking branch 'upstream/master' into PHOENIX-3534


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7420ae3a
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7420ae3a
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7420ae3a

Branch: refs/heads/system-catalog
Commit: 7420ae3aaf6c8a02c17dedb155a29bbd9d333203
Parents: 0f58bc9 d10151e
Author: Thomas D'Silva <td...@apache.org>
Authored: Mon Apr 30 10:39:24 2018 -0700
Committer: Thomas D'Silva <td...@apache.org>
Committed: Mon Apr 30 10:39:24 2018 -0700

----------------------------------------------------------------------
 .../src/build/components/all-common-files.xml   |   5 +
 phoenix-assembly/src/build/src.xml              |   7 +-
 .../apache/phoenix/end2end/AlterTableIT.java    |  12 +-
 .../phoenix/end2end/AlterTableWithViewsIT.java  |  22 ++
 .../end2end/ExplainPlanWithStatsEnabledIT.java  |  49 +++-
 .../phoenix/tx/ParameterizedTransactionIT.java  |  37 +--
 .../coprocessor/MetaDataEndpointImpl.java       |  28 +-
 .../phoenix/coprocessor/MetaDataProtocol.java   |   1 -
 .../org/apache/phoenix/execute/ScanPlan.java    |   4 +-
 .../phoenix/iterate/BaseResultIterators.java    |  50 +++-
 .../phoenix/iterate/ParallelIterators.java      |   8 +
 .../apache/phoenix/iterate/SerialIterators.java |  13 +-
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |   1 +
 .../query/ConnectionQueryServicesImpl.java      |  24 +-
 .../query/ConnectionlessQueryServicesImpl.java  |   7 +-
 .../apache/phoenix/query/QueryConstants.java    | 107 ++++----
 .../apache/phoenix/schema/MetaDataClient.java   |  22 +-
 .../SplitOnLeadingVarCharColumnsPolicy.java     |  43 +++
 .../schema/SystemFunctionSplitPolicy.java       |  27 ++
 .../phoenix/schema/SystemStatsSplitPolicy.java  |  27 ++
 .../apache/phoenix/schema/TableProperty.java    | 270 ++++++++++---------
 .../phoenix/schema/stats/StatisticsUtil.java    |   6 +
 .../phoenix/schema/SystemSplitPolicyTest.java   |  97 +++++++
 .../apache/phoenix/hive/HivePhoenixStoreIT.java |  27 ++
 .../PhoenixStorageHandlerConstants.java         |   8 +-
 .../phoenix/hive/query/PhoenixQueryBuilder.java |   4 +-
 .../pherf/configuration/XMLConfigParser.java    |   2 +-
 .../pherf/result/impl/XMLResultHandler.java     |  18 +-
 28 files changed, 664 insertions(+), 262 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/7420ae3a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7420ae3a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
index 1d6e786,e1b1372..0e926f0
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
@@@ -40,14 -40,14 +41,16 @@@ import org.apache.hadoop.hbase.util.Byt
  import org.apache.phoenix.coprocessor.TephraTransactionalProcessor;
  import org.apache.phoenix.exception.SQLExceptionCode;
  import org.apache.phoenix.jdbc.PhoenixConnection;
+ import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
  import org.apache.phoenix.query.QueryConstants;
 +import org.apache.phoenix.schema.PColumn;
  import org.apache.phoenix.schema.PName;
  import org.apache.phoenix.schema.PNameFactory;
  import org.apache.phoenix.schema.PTable;
  import org.apache.phoenix.schema.PTableKey;
  import org.apache.phoenix.schema.PTableType;
 +import org.apache.phoenix.util.PhoenixRuntime;
+ import org.apache.phoenix.util.StringUtil;
  import org.junit.Test;
  import org.junit.runner.RunWith;
  import org.junit.runners.Parameterized;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7420ae3a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7420ae3a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 06c9aa9,b77f113..454bcda
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@@ -584,8 -591,9 +588,8 @@@ public class MetaDataEndpointImpl exten
                  builder.setTable(PTableImpl.toProto(table));
              }
              done.run(builder.build());
 -            return;
          } catch (Throwable t) {
-         	logger.error("getTable failed", t);
+             logger.error("getTable failed", t);
              ProtobufUtil.setControllerException(controller,
                  ServerUtil.createIOException(SchemaUtil.getTableName(schemaName, tableName), t));
          }
@@@ -2807,32 -2544,45 +2810,43 @@@
                  }
              }
          }
 -        // Sort the puts by ordinal position 
 +        // Sort the puts by ordinal position
          Collections.sort(columnPutsForBaseTable);
 -        for (ViewInfo viewInfo : childViewsResult.getViewInfoList()) {
 -            short deltaNumPkColsSoFar = 0;
 -            short columnsAddedToView = 0;
 -            short columnsAddedToBaseTable = 0;
 +        for (TableInfo viewInfo : childViewsResult.getResults()) {
+             byte[] tenantId = viewInfo.getTenantId();
+             byte[] schema = viewInfo.getSchemaName();
 -            byte[] table = viewInfo.getViewName();
 -            byte[] viewKey = SchemaUtil.getTableKey(tenantId, schema, table);
 -            
 -            // lock the rows corresponding to views so that no other thread can modify the view meta-data
 -            RowLock viewRowLock = acquireLock(region, viewKey, locks);
 -            PTable view = doGetTable(viewKey, clientTimeStamp, viewRowLock, clientVersion);
++            byte[] table = viewInfo.getTableName();
 +            byte[] viewKey = SchemaUtil.getTableKey(viewInfo.getTenantId(), viewInfo.getSchemaName(), viewInfo.getTableName());
 +            PTable view = doGetTable(viewKey, clientTimeStamp, clientVersion);
+             if (view == null) {
+                 logger.warn("Found orphan tenant view row in SYSTEM.CATALOG with tenantId:"
+                     + Bytes.toString(tenantId) + ", schema:"
+                     + Bytes.toString(schema) + ", table:"
+                     + Bytes.toString(table));
+                 continue;
+              }
+             
 -            ColumnOrdinalPositionUpdateList ordinalPositionList = new ColumnOrdinalPositionUpdateList();
 +
 +            // add the new columns to the child view
              List<PColumn> viewPkCols = new ArrayList<>(view.getPKColumns());
              boolean addingExistingPkCol = false;
 -            int numCols = view.getColumns().size();
 -            // add the new columns to the child view
              for (PutWithOrdinalPosition p : columnPutsForBaseTable) {
 -                Put baseTableColumnPut = p.put;
 +                Put columnToBeAdded = p.put;
                  PColumn existingViewColumn = null;
                  byte[][] rkmd = new byte[5][];
 -                getVarChars(baseTableColumnPut.getRow(), rkmd);
 +                getVarChars(columnToBeAdded.getRow(), rkmd);
                  String columnName = Bytes.toString(rkmd[COLUMN_NAME_INDEX]);
 -                String columnFamily = rkmd[FAMILY_NAME_INDEX] == null ? null : Bytes.toString(rkmd[FAMILY_NAME_INDEX]);
 +                String columnFamily =
 +                        rkmd[FAMILY_NAME_INDEX] == null ? null
 +                                : Bytes.toString(rkmd[FAMILY_NAME_INDEX]);
                  try {
 -                    existingViewColumn = columnFamily == null ? view.getColumnForColumnName(columnName) : view.getColumnFamily(
 -                            columnFamily).getPColumnForColumnName(columnName);
 +                    existingViewColumn =
 +                            columnFamily == null ? view.getColumnForColumnName(columnName)
 +                                    : view.getColumnFamily(columnFamily)
 +                                            .getPColumnForColumnName(columnName);
                  } catch (ColumnFamilyNotFoundException e) {
 -                    // ignore since it means that the column family is not present for the column to be added.
 +                    // ignore since it means that the column family is not present for the column to
 +                    // be added.
                  } catch (ColumnNotFoundException e) {
                      // ignore since it means the column is not present in the view
                  }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7420ae3a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7420ae3a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7420ae3a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 45016c6,1d3feed..448382a
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@@ -2471,13 -2471,8 +2473,13 @@@ public class ConnectionQueryServicesImp
  
      // Available for testing
      protected String getLogTableDDL() {
-         return setSystemLogDDLProperties(QueryConstants.CREATE_LOG_METADATA);
+         return QueryConstants.CREATE_LOG_METADATA;
      }
 +    
 +    // Available for testing
 +    protected String getChildLinkDDL() {
 +        return setSystemDDLProperties(QueryConstants.CREATE_CHILD_LINK_METADATA);
 +    }
  
      private String setSystemDDLProperties(String ddl) {
          return String.format(ddl,
@@@ -2964,12 -2951,22 +2961,27 @@@
                MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0,
                PhoenixDatabaseMetaData.TRANSACTION_PROVIDER + " "
                  + PTinyint.INSTANCE.getSqlTypeName());
+             metaConnection.createStatement().executeUpdate("ALTER TABLE " + 
+                     PhoenixDatabaseMetaData.SYSTEM_CATALOG + " SET " + 
+                     HConstants.VERSIONS + "= " + props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS) + ",\n" +
+                     HColumnDescriptor.KEEP_DELETED_CELLS + "=" + props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS)
+                     );
+             metaConnection.createStatement().executeUpdate("ALTER TABLE " + 
+                 PhoenixDatabaseMetaData.SYSTEM_FUNCTION + " SET " + 
+                     HTableDescriptor.SPLIT_POLICY + "='" + SystemFunctionSplitPolicy.class.getName() + "',\n" +
+                     HConstants.VERSIONS + "= " + props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS) + ",\n" +
+                     HColumnDescriptor.KEEP_DELETED_CELLS + "=" + props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS)
+                     );
+             metaConnection.createStatement().executeUpdate("ALTER TABLE " + 
+                     PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET " + 
+                     HTableDescriptor.SPLIT_POLICY + "='" + SystemStatsSplitPolicy.class.getName() +"'"
+                     );
          }
 +        // TODO set the version for which the following upgrade code runs correct
 +        if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0) {
 +            addViewIndexToParentLinks(metaConnection);
 +            moveChildLinks(metaConnection);
 +        }
          return metaConnection;
      }
  

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7420ae3a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 0fdf5e6,3531a87..11e33e9
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@@ -164,12 -164,8 +164,12 @@@ public class ConnectionlessQueryService
      }
  
      protected String getLogTableDDL() {
-         return setSystemLogDDLProperties(QueryConstants.CREATE_LOG_METADATA);
+         return QueryConstants.CREATE_LOG_METADATA;
      }
 +    
 +    protected String getChildLinkDDL() {
 +        return setSystemDDLProperties(QueryConstants.CREATE_CHILD_LINK_METADATA);
 +    }
  
      private String setSystemDDLProperties(String ddl) {
          return String.format(ddl,

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7420ae3a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index aa37517,2fe7b14..97acba0
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@@ -10,7 -17,113 +10,8 @@@
   */
  package org.apache.phoenix.query;
  
+ 
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.AUTO_PARTITION_SEQ;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BASE_COLUMN_COUNT;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BIND_PARAMETERS;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BUFFER_LENGTH;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CACHE_SIZE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CHAR_OCTET_LENGTH;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLASS_NAME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLIENT_IP;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CURRENT_VALUE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CYCLE_FLAG;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ENCODING_SCHEME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXCEPTION_TRACE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXPLAIN_PLAN;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GLOBAL_SCAN_DETAILS;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POST_KEY;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INCREMENT_BY;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_ARRAY;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_AUTOINCREMENT;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_CONSTANT;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_NAMESPACE_MAPPED;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_NULLABLE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_ROW_TIMESTAMP;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_VIEW_REFERENCED;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.JAR_PATH;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.KEY_SEQ;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LAST_STATS_UPDATE_TIME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LIMIT_REACHED_FLAG;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MAX_VALUE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MIN_VALUE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MULTI_TENANT;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NO_OF_RESULTS_ITERATED;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NULLABLE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_ARGS;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_PREC_RADIX;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PHYSICAL_NAME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PK_NAME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_ID;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_STATUS;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.REF_GENERATION;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.REMARKS;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.RETURN_TYPE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SALT_BUCKETS;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCAN_METRICS_JSON;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_CATALOG;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_SCHEMA;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_TABLE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SELF_REFERENCING_COL_NAME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SEQUENCE_NAME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SEQUENCE_SCHEMA;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SOURCE_DATA_TYPE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATA_TYPE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATETIME_SUB;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_TIME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_WITH;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STORE_NULLS;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_TABLE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TOTAL_EXECUTION_TIME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTION_PROVIDER;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_NAME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USER;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
 +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.*;
  
  import java.math.BigDecimal;
  
@@@ -26,10 -139,16 +27,12 @@@ import org.apache.phoenix.schema.PName
  import org.apache.phoenix.schema.PNameFactory;
  import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
  import org.apache.phoenix.schema.SortOrder;
+ import org.apache.phoenix.schema.SystemFunctionSplitPolicy;
+ import org.apache.phoenix.schema.SystemStatsSplitPolicy;
  import org.apache.phoenix.schema.TableProperty;
  
 -
  /**
 - *
   * Constants used during querying
 - *
 - *
   * @since 0.1
   */
  public interface QueryConstants {
@@@ -219,27 -354,35 +222,30 @@@
      public static final String CREATE_STATS_TABLE_METADATA =
              "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_STATS_TABLE + "\"(\n" +
              // PK columns
-                     PHYSICAL_NAME + " VARCHAR NOT NULL," + COLUMN_FAMILY + " VARCHAR,"
-                     + GUIDE_POST_KEY + " VARBINARY," + GUIDE_POSTS_WIDTH + " BIGINT,"
-                     + LAST_STATS_UPDATE_TIME + " DATE, " + GUIDE_POSTS_ROW_COUNT + " BIGINT, "
-                     + "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + PHYSICAL_NAME + ","
-                     + COLUMN_FAMILY + "," + GUIDE_POST_KEY + "))\n" +
-                     // Install split policy to prevent a physical table's stats from being split
-                     // across regions.
-                     HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName()
-                     + "',\n" + PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
+             PHYSICAL_NAME  + " VARCHAR NOT NULL," +
+             COLUMN_FAMILY + " VARCHAR," +
+             GUIDE_POST_KEY  + " VARBINARY," +
+             GUIDE_POSTS_WIDTH + " BIGINT," +
+             LAST_STATS_UPDATE_TIME+ " DATE, "+
+             GUIDE_POSTS_ROW_COUNT+ " BIGINT, "+
+             "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY ("
+             + PHYSICAL_NAME + ","
+             + COLUMN_FAMILY + ","+ GUIDE_POST_KEY+"))\n" +
+             // Install split policy to prevent a physical table's stats from being split across regions.
+             HTableDescriptor.SPLIT_POLICY + "='" + SystemStatsSplitPolicy.class.getName() + "',\n" + 
+             PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
  
      public static final String CREATE_SEQUENCE_METADATA =
 -            "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + TYPE_SEQUENCE + "\"(\n" +
 -            TENANT_ID + " VARCHAR NULL," +
 -            SEQUENCE_SCHEMA + " VARCHAR NULL, \n" +
 -            SEQUENCE_NAME +  " VARCHAR NOT NULL, \n" +
 -            START_WITH + " BIGINT, \n" +
 -            CURRENT_VALUE + " BIGINT, \n" +
 -            INCREMENT_BY  + " BIGINT, \n" +
 -            CACHE_SIZE  + " BIGINT, \n" +
 -            //  the following three columns were added in 3.1/4.1
 -            MIN_VALUE + " BIGINT, \n" +
 -            MAX_VALUE + " BIGINT, \n" +
 -            CYCLE_FLAG + " BOOLEAN, \n" +
 -            LIMIT_REACHED_FLAG + " BOOLEAN \n" +
 -            " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + SEQUENCE_SCHEMA + "," + SEQUENCE_NAME + "))\n" +
 -            PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
 +            "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + TYPE_SEQUENCE + "\"(\n" + TENANT_ID
 +                    + " VARCHAR NULL," + SEQUENCE_SCHEMA + " VARCHAR NULL, \n" + SEQUENCE_NAME
 +                    + " VARCHAR NOT NULL, \n" + START_WITH + " BIGINT, \n" + CURRENT_VALUE
 +                    + " BIGINT, \n" + INCREMENT_BY + " BIGINT, \n" + CACHE_SIZE + " BIGINT, \n" +
 +                    // the following three columns were added in 3.1/4.1
 +                    MIN_VALUE + " BIGINT, \n" + MAX_VALUE + " BIGINT, \n" + CYCLE_FLAG
 +                    + " BOOLEAN, \n" + LIMIT_REACHED_FLAG + " BOOLEAN \n" + " CONSTRAINT "
 +                    + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + SEQUENCE_SCHEMA
 +                    + "," + SEQUENCE_NAME + "))\n" + PhoenixDatabaseMetaData.TRANSACTIONAL + "="
 +                    + Boolean.FALSE;
      public static final String CREATE_SYSTEM_SCHEMA = "CREATE SCHEMA " + SYSTEM_CATALOG_SCHEMA;
      public static final String UPGRADE_TABLE_SNAPSHOT_PREFIX = "_UPGRADING_TABLE_";
  

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7420ae3a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------


[27/50] [abbrv] phoenix git commit: PHOENIX-4613 Thread clientVersion through to IndexCommitter implementors

Posted by td...@apache.org.
PHOENIX-4613 Thread clientVersion through to IndexCommitter implementors


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ccd41de7
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ccd41de7
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ccd41de7

Branch: refs/heads/system-catalog
Commit: ccd41de7bfd5047511db3eb6c17121e92eebf1e6
Parents: b768900
Author: James Taylor <jt...@salesforce.com>
Authored: Fri Apr 20 11:44:13 2018 -0700
Committer: Vincent Poon <vi...@apache.org>
Committed: Fri Apr 20 11:45:06 2018 -0700

----------------------------------------------------------------------
 .../org/apache/phoenix/hbase/index/Indexer.java | 14 +++++++++----
 .../hbase/index/builder/IndexBuildManager.java  |  4 ++++
 .../hbase/index/covered/IndexMetaData.java      | 12 ++++++++++-
 .../hbase/index/write/IndexCommitter.java       |  2 +-
 .../phoenix/hbase/index/write/IndexWriter.java  | 21 ++++++++++----------
 .../write/ParallelWriterIndexCommitter.java     |  2 +-
 .../hbase/index/write/RecoveryIndexWriter.java  |  4 ++--
 .../TrackingParallelWriterIndexCommitter.java   |  2 +-
 .../index/PhoenixTransactionalIndexer.java      |  9 +++++++--
 .../index/covered/LocalTableStateTest.java      | 18 ++++++++++++++++-
 .../hbase/index/write/TestIndexWriter.java      |  7 +++----
 .../index/write/TestParalleIndexWriter.java     |  4 +++-
 .../write/TestParalleWriterIndexCommitter.java  |  3 ++-
 13 files changed, 73 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/ccd41de7/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
index 7b9bc1f..7325cd8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
@@ -92,6 +92,7 @@ import org.apache.phoenix.trace.TracingUtils;
 import org.apache.phoenix.trace.util.NullSpan;
 import org.apache.phoenix.util.EnvironmentEdgeManager;
 import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.ServerUtil;
 
 import com.google.common.collect.Lists;
@@ -132,8 +133,13 @@ public class Indexer extends BaseRegionObserver {
   // Hack to get around not being able to save any state between
   // coprocessor calls. TODO: remove after HBASE-18127 when available
   private static class BatchMutateContext {
+      public final int clientVersion;
       public Collection<Pair<Mutation, byte[]>> indexUpdates = Collections.emptyList();
       public List<RowLock> rowLocks = Lists.newArrayListWithExpectedSize(QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE);
+
+      public BatchMutateContext(int clientVersion) {
+          this.clientVersion = clientVersion;
+      }
   }
   
   private ThreadLocal<BatchMutateContext> batchMutateContext =
@@ -396,7 +402,7 @@ public class Indexer extends BaseRegionObserver {
        * Exclusively lock all rows so we get a consistent read
        * while determining the index updates
        */
-      BatchMutateContext context = new BatchMutateContext();
+      BatchMutateContext context = new BatchMutateContext(this.builder.getIndexMetaData(miniBatchOp).getClientVersion());
       setBatchMutateContext(c, context);
       Durability durability = Durability.SKIP_WAL;
       boolean copyMutations = false;
@@ -624,7 +630,7 @@ public class Indexer extends BaseRegionObserver {
           long start = EnvironmentEdgeManager.currentTimeMillis();
           
           current.addTimelineAnnotation("Actually doing index update for first time");
-          writer.writeAndKillYourselfOnFailure(context.indexUpdates, false);
+          writer.writeAndKillYourselfOnFailure(context.indexUpdates, false, context.clientVersion);
 
           long duration = EnvironmentEdgeManager.currentTimeMillis() - start;
           if (duration >= slowIndexWriteThreshold) {
@@ -693,7 +699,7 @@ public class Indexer extends BaseRegionObserver {
         // do the usual writer stuff, killing the server again, if we can't manage to make the index
         // writes succeed again
         try {
-            writer.writeAndKillYourselfOnFailure(updates, true);
+            writer.writeAndKillYourselfOnFailure(updates, true, ScanUtil.UNKNOWN_CLIENT_VERSION);
         } catch (IOException e) {
                 LOG.error("During WAL replay of outstanding index updates, "
                         + "Exception is thrown instead of killing server during index writing", e);
@@ -731,7 +737,7 @@ public class Indexer extends BaseRegionObserver {
            * hopes they come up before the primary table finishes.
            */
           Collection<Pair<Mutation, byte[]>> indexUpdates = extractIndexUpdate(logEdit);
-          recoveryWriter.writeAndKillYourselfOnFailure(indexUpdates, true);
+          recoveryWriter.writeAndKillYourselfOnFailure(indexUpdates, true, ScanUtil.UNKNOWN_CLIENT_VERSION);
       } finally {
           long duration = EnvironmentEdgeManager.currentTimeMillis() - start;
           if (duration >= slowPreWALRestoreThreshold) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ccd41de7/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/builder/IndexBuildManager.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/builder/IndexBuildManager.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/builder/IndexBuildManager.java
index 4c410ad..2550dd1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/builder/IndexBuildManager.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/builder/IndexBuildManager.java
@@ -73,6 +73,10 @@ public class IndexBuildManager implements Stoppable {
     }
   }
 
+  public IndexMetaData getIndexMetaData(MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
+      return this.delegate.getIndexMetaData(miniBatchOp);
+  }
+
   public Collection<Pair<Mutation, byte[]>> getIndexUpdate(
       MiniBatchOperationInProgress<Mutation> miniBatchOp,
       Collection<? extends Mutation> mutations) throws Throwable {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ccd41de7/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/IndexMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/IndexMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/IndexMetaData.java
index 20ed855..18b515c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/IndexMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/IndexMetaData.java
@@ -19,8 +19,10 @@ package org.apache.phoenix.hbase.index.covered;
 
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.phoenix.coprocessor.BaseScannerRegionObserver.ReplayWrite;
+import org.apache.phoenix.util.ScanUtil;
 
 public interface IndexMetaData {
+
     public static final IndexMetaData NULL_INDEX_META_DATA = new IndexMetaData() {
 
         @Override
@@ -31,7 +33,13 @@ public interface IndexMetaData {
         @Override
         public ReplayWrite getReplayWrite() {
           return null;
-        }};
+        }
+
+        @Override
+        public int getClientVersion() {
+            return ScanUtil.UNKNOWN_CLIENT_VERSION;
+        }
+    };
 
         
     /**
@@ -42,4 +50,6 @@ public interface IndexMetaData {
     public boolean requiresPriorRowState(Mutation m);
 
     public ReplayWrite getReplayWrite();
+    
+    public int getClientVersion();
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ccd41de7/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexCommitter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexCommitter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexCommitter.java
index 5e3f3ed..e9dc202 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexCommitter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexCommitter.java
@@ -32,6 +32,6 @@ public interface IndexCommitter extends Stoppable {
 
   void setup(IndexWriter parent, RegionCoprocessorEnvironment env, String name);
 
-  public void write(Multimap<HTableInterfaceReference, Mutation> toWrite, boolean allowLocalUpdates)
+  public void write(Multimap<HTableInterfaceReference, Mutation> toWrite, boolean allowLocalUpdates, int clientVersion)
       throws IndexWriteException;
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ccd41de7/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriter.java
index 4e5e182..c28288c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriter.java
@@ -135,13 +135,14 @@ public class IndexWriter implements Stoppable {
    * which ensures that the server crashes when an index write fails, ensuring that we get WAL
    * replay of the index edits.
    * @param indexUpdates Updates to write
+ * @param clientVersion version of the client
  * @throws IOException 
    */
     public void writeAndKillYourselfOnFailure(Collection<Pair<Mutation, byte[]>> indexUpdates,
-            boolean allowLocalUpdates) throws IOException {
+            boolean allowLocalUpdates, int clientVersion) throws IOException {
     // convert the strings to htableinterfaces to which we can talk and group by TABLE
     Multimap<HTableInterfaceReference, Mutation> toWrite = resolveTableReferences(indexUpdates);
-    writeAndKillYourselfOnFailure(toWrite, allowLocalUpdates);
+    writeAndKillYourselfOnFailure(toWrite, allowLocalUpdates, clientVersion);
   }
 
   /**
@@ -150,9 +151,9 @@ public class IndexWriter implements Stoppable {
  * @throws IOException 
    */
     public void writeAndKillYourselfOnFailure(Multimap<HTableInterfaceReference, Mutation> toWrite,
-            boolean allowLocalUpdates) throws IOException {
+            boolean allowLocalUpdates, int clientVersion) throws IOException {
     try {
-      write(toWrite, allowLocalUpdates);
+      write(toWrite, allowLocalUpdates, clientVersion);
       if (LOG.isTraceEnabled()) {
         LOG.trace("Done writing all index updates!\n\t" + toWrite);
       }
@@ -176,12 +177,12 @@ public class IndexWriter implements Stoppable {
    * @throws IndexWriteException if we cannot successfully write to the index. Whether or not we
    *           stop early depends on the {@link IndexCommitter}.
    */
-    public void write(Collection<Pair<Mutation, byte[]>> toWrite) throws IndexWriteException {
-    	write(resolveTableReferences(toWrite), false);
+    public void write(Collection<Pair<Mutation, byte[]>> toWrite, int clientVersion) throws IndexWriteException {
+    	write(resolveTableReferences(toWrite), false, clientVersion);
     }
 
-    public void write(Collection<Pair<Mutation, byte[]>> toWrite, boolean allowLocalUpdates) throws IOException {
-    	write(resolveTableReferences(toWrite), allowLocalUpdates);
+    public void write(Collection<Pair<Mutation, byte[]>> toWrite, boolean allowLocalUpdates, int clientVersion) throws IOException {
+    	write(resolveTableReferences(toWrite), allowLocalUpdates, clientVersion);
     }
     
     /**
@@ -189,9 +190,9 @@ public class IndexWriter implements Stoppable {
    * @param toWrite
    * @throws IndexWriteException
    */
-  public void write(Multimap<HTableInterfaceReference, Mutation> toWrite, boolean allowLocalUpdates)
+  public void write(Multimap<HTableInterfaceReference, Mutation> toWrite, boolean allowLocalUpdates, int clientVersion)
 	      throws IndexWriteException {
-	  this.writer.write(toWrite, allowLocalUpdates);
+	  this.writer.write(toWrite, allowLocalUpdates, clientVersion);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ccd41de7/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
index 0bb8784..aba2678 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
@@ -94,7 +94,7 @@ public class ParallelWriterIndexCommitter implements IndexCommitter {
     }
 
     @Override
-    public void write(Multimap<HTableInterfaceReference, Mutation> toWrite, final boolean allowLocalUpdates) throws SingleIndexWriteFailureException {
+    public void write(Multimap<HTableInterfaceReference, Mutation> toWrite, final boolean allowLocalUpdates, final int clientVersion) throws SingleIndexWriteFailureException {
         /*
          * This bit here is a little odd, so let's explain what's going on. Basically, we want to do the writes in
          * parallel to each index table, so each table gets its own task and is submitted to the pool. Where it gets

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ccd41de7/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/RecoveryIndexWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/RecoveryIndexWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/RecoveryIndexWriter.java
index e340784..35f0a6d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/RecoveryIndexWriter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/RecoveryIndexWriter.java
@@ -69,9 +69,9 @@ public class RecoveryIndexWriter extends IndexWriter {
     }
 
     @Override
-    public void write(Collection<Pair<Mutation, byte[]>> toWrite, boolean allowLocalUpdates) throws IOException {
+    public void write(Collection<Pair<Mutation, byte[]>> toWrite, boolean allowLocalUpdates, int clientVersion) throws IOException {
         try {
-            write(resolveTableReferences(toWrite), allowLocalUpdates);
+            write(resolveTableReferences(toWrite), allowLocalUpdates, clientVersion);
         } catch (MultiIndexWriteFailureException e) {
             for (HTableInterfaceReference table : e.getFailedTables()) {
                 if (!admin.tableExists(table.getTableName())) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ccd41de7/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
index 94d4f0f..4dbad63 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
@@ -115,7 +115,7 @@ public class TrackingParallelWriterIndexCommitter implements IndexCommitter {
     }
 
     @Override
-    public void write(Multimap<HTableInterfaceReference, Mutation> toWrite, final boolean allowLocalUpdates) throws MultiIndexWriteFailureException {
+    public void write(Multimap<HTableInterfaceReference, Mutation> toWrite, final boolean allowLocalUpdates, final int clientVersion) throws MultiIndexWriteFailureException {
         Set<Entry<HTableInterfaceReference, Collection<Mutation>>> entries = toWrite.asMap().entrySet();
         TaskBatch<Boolean> tasks = new TaskBatch<Boolean>(entries.size());
         List<HTableInterfaceReference> tables = new ArrayList<HTableInterfaceReference>(entries.size());

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ccd41de7/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
index f0b2678..bdfcaff 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
@@ -74,6 +74,11 @@ public class PhoenixTransactionalIndexer extends BaseRegionObserver {
     // coprocessor calls. TODO: remove after HBASE-18127 when available
     private static class BatchMutateContext {
         public Collection<Pair<Mutation, byte[]>> indexUpdates = Collections.emptyList();
+        public final int clientVersion;
+
+        public BatchMutateContext(int clientVersion) {
+            this.clientVersion = clientVersion;
+        }
     }
     
     private ThreadLocal<BatchMutateContext> batchMutateContext =
@@ -159,7 +164,7 @@ public class PhoenixTransactionalIndexer extends BaseRegionObserver {
             super.preBatchMutate(c, miniBatchOp);
             return;
         }
-        BatchMutateContext context = new BatchMutateContext();
+        BatchMutateContext context = new BatchMutateContext(indexMetaData.getClientVersion());
         setBatchMutateContext(c, context);
         
         Collection<Pair<Mutation, byte[]>> indexUpdates = null;
@@ -229,7 +234,7 @@ public class PhoenixTransactionalIndexer extends BaseRegionObserver {
 
             if (success) { // if miniBatchOp was successfully written, write index updates
                 if (!context.indexUpdates.isEmpty()) {
-                    this.writer.write(context.indexUpdates, false);
+                    this.writer.write(context.indexUpdates, false, context.clientVersion);
                 }
                 current.addTimelineAnnotation("Wrote index updates");
             }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ccd41de7/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/LocalTableStateTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/LocalTableStateTest.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/LocalTableStateTest.java
index c7e1769..56ba1d6 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/LocalTableStateTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/LocalTableStateTest.java
@@ -40,6 +40,7 @@ import org.apache.phoenix.hbase.index.covered.data.LocalTable;
 import org.apache.phoenix.hbase.index.covered.update.ColumnReference;
 import org.apache.phoenix.hbase.index.scanner.Scanner;
 import org.apache.phoenix.hbase.index.scanner.ScannerBuilder.CoveredDeleteScanner;
+import org.apache.phoenix.util.ScanUtil;
 import org.junit.Test;
 import org.mockito.Mockito;
 import org.mockito.invocation.InvocationOnMock;
@@ -67,6 +68,11 @@ public class LocalTableStateTest {
         return true;
     }
       
+    @Override
+    public int getClientVersion() {
+        return ScanUtil.UNKNOWN_CLIENT_VERSION;
+    }
+
   };
 
   @SuppressWarnings("unchecked")
@@ -130,7 +136,12 @@ public class LocalTableStateTest {
             return true;
         }
             
-        };
+        @Override
+        public int getClientVersion() {
+            return ScanUtil.UNKNOWN_CLIENT_VERSION;
+        }
+
+    };
     Put m = new Put(row);
     m.addColumn(fam, qual, ts, val);
     // setup mocks
@@ -167,6 +178,11 @@ public class LocalTableStateTest {
             return false;
         }
             
+        @Override
+        public int getClientVersion() {
+            return ScanUtil.UNKNOWN_CLIENT_VERSION;
+        }
+
     };
     Put m = new Put(row);
     m.addColumn(fam, qual, ts, val);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ccd41de7/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestIndexWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestIndexWriter.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestIndexWriter.java
index a25f7cf..58050c1 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestIndexWriter.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestIndexWriter.java
@@ -17,10 +17,8 @@
  */
 package org.apache.phoenix.hbase.index.write;
 
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -51,6 +49,7 @@ import org.apache.phoenix.hbase.index.StubAbortable;
 import org.apache.phoenix.hbase.index.TableName;
 import org.apache.phoenix.hbase.index.exception.IndexWriteException;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.util.ScanUtil;
 import org.junit.Rule;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -134,7 +133,7 @@ public class TestIndexWriter {
     KillServerOnFailurePolicy policy = new KillServerOnFailurePolicy();
     policy.setup(stop, abort);
     IndexWriter writer = new IndexWriter(committer, policy);
-    writer.write(indexUpdates);
+    writer.write(indexUpdates, ScanUtil.UNKNOWN_CLIENT_VERSION);
     assertTrue("Writer returned before the table batch completed! Likely a race condition tripped",
       completed[0]);
     writer.stop(this.testName.getTableNameString() + " finished");
@@ -208,7 +207,7 @@ public class TestIndexWriter {
       @Override
       public void run() {
         try {
-          writer.write(indexUpdates);
+          writer.write(indexUpdates, ScanUtil.UNKNOWN_CLIENT_VERSION);
         } catch (IndexWriteException e) {
           failedWrite[0] = true;
         }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ccd41de7/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleIndexWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleIndexWriter.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleIndexWriter.java
index cd29e10..55c3fb3 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleIndexWriter.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleIndexWriter.java
@@ -41,8 +41,10 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.VersionInfo;
 import org.apache.phoenix.hbase.index.StubAbortable;
 import org.apache.phoenix.hbase.index.TableName;
+import org.apache.phoenix.hbase.index.covered.IndexMetaData;
 import org.apache.phoenix.hbase.index.table.HTableInterfaceReference;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.util.ScanUtil;
 import org.junit.Rule;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -124,7 +126,7 @@ public class TestParalleIndexWriter {
     // setup the writer and failure policy
     TrackingParallelWriterIndexCommitter writer = new TrackingParallelWriterIndexCommitter(VersionInfo.getVersion());
     writer.setup(factory, exec, abort, stop, e);
-    writer.write(indexUpdates, true);
+    writer.write(indexUpdates, true, ScanUtil.UNKNOWN_CLIENT_VERSION);
     assertTrue("Writer returned before the table batch completed! Likely a race condition tripped",
       completed[0]);
     writer.stop(this.test.getTableNameString() + " finished");

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ccd41de7/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleWriterIndexCommitter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleWriterIndexCommitter.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleWriterIndexCommitter.java
index 32ae108..9767eae 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleWriterIndexCommitter.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleWriterIndexCommitter.java
@@ -43,6 +43,7 @@ import org.apache.phoenix.hbase.index.StubAbortable;
 import org.apache.phoenix.hbase.index.TableName;
 import org.apache.phoenix.hbase.index.table.HTableInterfaceReference;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.util.ScanUtil;
 import org.junit.Rule;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -125,7 +126,7 @@ public class TestParalleWriterIndexCommitter {
     // setup the writer and failure policy
     TrackingParallelWriterIndexCommitter writer = new TrackingParallelWriterIndexCommitter(VersionInfo.getVersion());
     writer.setup(factory, exec, abort, stop, e);
-    writer.write(indexUpdates, true);
+    writer.write(indexUpdates, true, ScanUtil.UNKNOWN_CLIENT_VERSION);
     assertTrue("Writer returned before the table batch completed! Likely a race condition tripped",
       completed[0]);
     writer.stop(this.test.getTableNameString() + " finished");


[13/50] [abbrv] phoenix git commit: PHOENIX-4605 Support running multiple transaction providers

Posted by td...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java
index c5065e0..59c10ad 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java
@@ -19,120 +19,7 @@ package org.apache.phoenix.expression;
 
 import java.util.Map;
 
-import org.apache.phoenix.expression.function.AbsFunction;
-import org.apache.phoenix.expression.function.ArrayAllComparisonExpression;
-import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression;
-import org.apache.phoenix.expression.function.ArrayAppendFunction;
-import org.apache.phoenix.expression.function.ArrayConcatFunction;
-import org.apache.phoenix.expression.function.ArrayElemRefExpression;
-import org.apache.phoenix.expression.function.ArrayFillFunction;
-import org.apache.phoenix.expression.function.ArrayIndexFunction;
-import org.apache.phoenix.expression.function.ArrayLengthFunction;
-import org.apache.phoenix.expression.function.ArrayPrependFunction;
-import org.apache.phoenix.expression.function.ArrayRemoveFunction;
-import org.apache.phoenix.expression.function.ArrayToStringFunction;
-import org.apache.phoenix.expression.function.ByteBasedRegexpReplaceFunction;
-import org.apache.phoenix.expression.function.ByteBasedRegexpSplitFunction;
-import org.apache.phoenix.expression.function.ByteBasedRegexpSubstrFunction;
-import org.apache.phoenix.expression.function.CbrtFunction;
-import org.apache.phoenix.expression.function.CeilDateExpression;
-import org.apache.phoenix.expression.function.CeilDecimalExpression;
-import org.apache.phoenix.expression.function.CeilFunction;
-import org.apache.phoenix.expression.function.CeilMonthExpression;
-import org.apache.phoenix.expression.function.CeilTimestampExpression;
-import org.apache.phoenix.expression.function.CeilWeekExpression;
-import org.apache.phoenix.expression.function.CeilYearExpression;
-import org.apache.phoenix.expression.function.CoalesceFunction;
-import org.apache.phoenix.expression.function.CollationKeyFunction;
-import org.apache.phoenix.expression.function.ConvertTimezoneFunction;
-import org.apache.phoenix.expression.function.CountAggregateFunction;
-import org.apache.phoenix.expression.function.DayOfMonthFunction;
-import org.apache.phoenix.expression.function.DayOfWeekFunction;
-import org.apache.phoenix.expression.function.DayOfYearFunction;
-import org.apache.phoenix.expression.function.DecodeFunction;
-import org.apache.phoenix.expression.function.DefaultValueExpression;
-import org.apache.phoenix.expression.function.DistinctCountAggregateFunction;
-import org.apache.phoenix.expression.function.DistinctCountHyperLogLogAggregateFunction;
-import org.apache.phoenix.expression.function.DistinctValueAggregateFunction;
-import org.apache.phoenix.expression.function.EncodeFunction;
-import org.apache.phoenix.expression.function.ExpFunction;
-import org.apache.phoenix.expression.function.ExternalSqlTypeIdFunction;
-import org.apache.phoenix.expression.function.FirstValueFunction;
-import org.apache.phoenix.expression.function.FirstValuesFunction;
-import org.apache.phoenix.expression.function.FloorDateExpression;
-import org.apache.phoenix.expression.function.FloorDecimalExpression;
-import org.apache.phoenix.expression.function.FloorFunction;
-import org.apache.phoenix.expression.function.FloorMonthExpression;
-import org.apache.phoenix.expression.function.FloorWeekExpression;
-import org.apache.phoenix.expression.function.FloorYearExpression;
-import org.apache.phoenix.expression.function.GetBitFunction;
-import org.apache.phoenix.expression.function.GetByteFunction;
-import org.apache.phoenix.expression.function.HourFunction;
-import org.apache.phoenix.expression.function.IndexStateNameFunction;
-import org.apache.phoenix.expression.function.InstrFunction;
-import org.apache.phoenix.expression.function.InvertFunction;
-import org.apache.phoenix.expression.function.LTrimFunction;
-import org.apache.phoenix.expression.function.LastValueFunction;
-import org.apache.phoenix.expression.function.LastValuesFunction;
-import org.apache.phoenix.expression.function.LengthFunction;
-import org.apache.phoenix.expression.function.LnFunction;
-import org.apache.phoenix.expression.function.LogFunction;
-import org.apache.phoenix.expression.function.LowerFunction;
-import org.apache.phoenix.expression.function.LpadFunction;
-import org.apache.phoenix.expression.function.MD5Function;
-import org.apache.phoenix.expression.function.MaxAggregateFunction;
-import org.apache.phoenix.expression.function.MinAggregateFunction;
-import org.apache.phoenix.expression.function.MinuteFunction;
-import org.apache.phoenix.expression.function.MonthFunction;
-import org.apache.phoenix.expression.function.NowFunction;
-import org.apache.phoenix.expression.function.NthValueFunction;
-import org.apache.phoenix.expression.function.OctetLengthFunction;
-import org.apache.phoenix.expression.function.PercentRankAggregateFunction;
-import org.apache.phoenix.expression.function.PercentileContAggregateFunction;
-import org.apache.phoenix.expression.function.PercentileDiscAggregateFunction;
-import org.apache.phoenix.expression.function.PowerFunction;
-import org.apache.phoenix.expression.function.RTrimFunction;
-import org.apache.phoenix.expression.function.RandomFunction;
-import org.apache.phoenix.expression.function.RegexpReplaceFunction;
-import org.apache.phoenix.expression.function.RegexpSplitFunction;
-import org.apache.phoenix.expression.function.RegexpSubstrFunction;
-import org.apache.phoenix.expression.function.ReverseFunction;
-import org.apache.phoenix.expression.function.RoundDateExpression;
-import org.apache.phoenix.expression.function.RoundDecimalExpression;
-import org.apache.phoenix.expression.function.RoundFunction;
-import org.apache.phoenix.expression.function.RoundMonthExpression;
-import org.apache.phoenix.expression.function.RoundTimestampExpression;
-import org.apache.phoenix.expression.function.RoundWeekExpression;
-import org.apache.phoenix.expression.function.RoundYearExpression;
-import org.apache.phoenix.expression.function.SQLIndexTypeFunction;
-import org.apache.phoenix.expression.function.SQLTableTypeFunction;
-import org.apache.phoenix.expression.function.SQLViewTypeFunction;
-import org.apache.phoenix.expression.function.SecondFunction;
-import org.apache.phoenix.expression.function.SetBitFunction;
-import org.apache.phoenix.expression.function.SetByteFunction;
-import org.apache.phoenix.expression.function.SignFunction;
-import org.apache.phoenix.expression.function.SqlTypeNameFunction;
-import org.apache.phoenix.expression.function.SqrtFunction;
-import org.apache.phoenix.expression.function.StddevPopFunction;
-import org.apache.phoenix.expression.function.StddevSampFunction;
-import org.apache.phoenix.expression.function.StringBasedRegexpReplaceFunction;
-import org.apache.phoenix.expression.function.StringBasedRegexpSplitFunction;
-import org.apache.phoenix.expression.function.StringBasedRegexpSubstrFunction;
-import org.apache.phoenix.expression.function.StringToArrayFunction;
-import org.apache.phoenix.expression.function.SubstrFunction;
-import org.apache.phoenix.expression.function.SumAggregateFunction;
-import org.apache.phoenix.expression.function.TimezoneOffsetFunction;
-import org.apache.phoenix.expression.function.ToCharFunction;
-import org.apache.phoenix.expression.function.ToDateFunction;
-import org.apache.phoenix.expression.function.ToNumberFunction;
-import org.apache.phoenix.expression.function.ToTimeFunction;
-import org.apache.phoenix.expression.function.ToTimestampFunction;
-import org.apache.phoenix.expression.function.TrimFunction;
-import org.apache.phoenix.expression.function.TruncFunction;
-import org.apache.phoenix.expression.function.UDFExpression;
-import org.apache.phoenix.expression.function.UpperFunction;
-import org.apache.phoenix.expression.function.WeekFunction;
-import org.apache.phoenix.expression.function.YearFunction;
+import org.apache.phoenix.expression.function.*;
 
 import com.google.common.collect.Maps;
 
@@ -298,7 +185,9 @@ public enum ExpressionType {
     LastValuesFunction(LastValuesFunction.class),
     DistinctCountHyperLogLogAggregateFunction(DistinctCountHyperLogLogAggregateFunction.class),
     CollationKeyFunction(CollationKeyFunction.class),
-    ArrayRemoveFunction(ArrayRemoveFunction.class);
+    ArrayRemoveFunction(ArrayRemoveFunction.class),
+    TransactionProviderNameFunction(TransactionProviderNameFunction.class),
+    ;
 
     ExpressionType(Class<? extends Expression> clazz) {
         this.clazz = clazz;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/expression/function/TransactionProviderNameFunction.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/TransactionProviderNameFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/TransactionProviderNameFunction.java
new file mode 100644
index 0000000..0117c1f
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/TransactionProviderNameFunction.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.expression.function;
+
+import java.sql.SQLException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.parse.FunctionParseNode.Argument;
+import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction;
+import org.apache.phoenix.schema.tuple.Tuple;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.schema.types.PTinyint;
+import org.apache.phoenix.schema.types.PVarchar;
+import org.apache.phoenix.transaction.TransactionFactory;
+
+
+/**
+ * 
+ * Function used to get the index state name from the serialized byte value
+ * Usage:
+ * IndexStateName('a')
+ * will return 'ACTIVE'
+ * 
+ * 
+ * @since 2.1
+ */
+@BuiltInFunction(name=TransactionProviderNameFunction.NAME, args= {
+    @Argument(allowedTypes= PInteger.class)} )
+public class TransactionProviderNameFunction extends ScalarFunction {
+    public static final String NAME = "TransactionProviderName";
+
+    public TransactionProviderNameFunction() {
+    }
+    
+    public TransactionProviderNameFunction(List<Expression> children) throws SQLException {
+        super(children);
+    }
+    
+    @Override
+    public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
+        Expression child = children.get(0);
+        if (!child.evaluate(tuple, ptr)) {
+            return false;
+        }
+        if (ptr.getLength() == 0) {
+            return true;
+        }
+        int code = PTinyint.INSTANCE.getCodec().decodeByte(ptr, child.getSortOrder());
+        TransactionFactory.Provider provider = TransactionFactory.Provider.fromCode(code);
+        ptr.set(PVarchar.INSTANCE.toBytes(provider.name()));
+        return true;
+    }
+
+    @Override
+    public PDataType getDataType() {
+        return PVarchar.INSTANCE;
+    }
+    
+    @Override
+    public String getName() {
+        return NAME;
+    }
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
index 15d8ac3..2f41dc3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
@@ -101,7 +101,6 @@ import org.apache.phoenix.schema.ValueSchema.Field;
 import org.apache.phoenix.schema.tuple.BaseTuple;
 import org.apache.phoenix.schema.tuple.ValueGetterTuple;
 import org.apache.phoenix.schema.types.PDataType;
-import org.apache.phoenix.transaction.TransactionFactory;
 import org.apache.phoenix.util.BitSet;
 import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.EncodedColumnsUtil;
@@ -109,6 +108,7 @@ import org.apache.phoenix.util.ExpressionUtil;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.MetaDataUtil;
 import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TransactionUtil;
 import org.apache.phoenix.util.TrustedByteArrayOutputStream;
 
 import com.google.common.base.Preconditions;
@@ -1068,7 +1068,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> {
             }
         	else if (kv.getTypeByte() == KeyValue.Type.DeleteFamily.getCode()
         			// Since we don't include the index rows in the change set for txn tables, we need to detect row deletes that have transformed by TransactionProcessor
-        			|| (CellUtil.matchingQualifier(kv, TransactionFactory.getTransactionProvider().getTransactionContext().getFamilyDeleteMarker()) && CellUtil.matchingValue(kv, HConstants.EMPTY_BYTE_ARRAY))) {
+        	        || TransactionUtil.isDeleteFamily(kv)) {
         	    nDeleteCF++;
         	}
         }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheFactory.java
index 94fbd0d..778401e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheFactory.java
@@ -52,7 +52,7 @@ public class IndexMetaDataCacheFactory implements ServerCacheFactory {
                 IndexMaintainer.deserialize(cachePtr, GenericKeyValueBuilder.INSTANCE, useProtoForIndexMaintainer);
         final PhoenixTransactionContext txnContext;
         try {
-            txnContext = txState.length != 0 ? TransactionFactory.getTransactionProvider().getTransactionContext(txState) : null;
+            txnContext = TransactionFactory.getTransactionContext(txState, clientVersion);
         } catch (IOException e) {
             throw new SQLException(e);
         }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexCodec.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexCodec.java b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexCodec.java
index c5233d3..d33e3fe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexCodec.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexCodec.java
@@ -44,7 +44,6 @@ public class PhoenixIndexCodec extends BaseIndexCodec {
     public static final String INDEX_PROTO_MD = "IdxProtoMD";
     public static final String INDEX_UUID = "IdxUUID";
     public static final String INDEX_MAINTAINERS = "IndexMaintainers";
-    public static final String CLIENT_VERSION = "_ClientVersion";
     public static KeyValueBuilder KV_BUILDER = GenericKeyValueBuilder.INSTANCE;
     
     private byte[] regionStartKey;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaDataBuilder.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaDataBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaDataBuilder.java
index 5e6f756..949e6ed 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaDataBuilder.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaDataBuilder.java
@@ -37,6 +37,7 @@ import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 import org.apache.phoenix.transaction.PhoenixTransactionContext;
 import org.apache.phoenix.transaction.TransactionFactory;
 import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.ServerUtil;
 
 public class PhoenixIndexMetaDataBuilder {
@@ -63,9 +64,9 @@ public class PhoenixIndexMetaDataBuilder {
             boolean useProto = md != null;
             byte[] txState = attributes.get(BaseScannerRegionObserver.TX_STATE);
             final List<IndexMaintainer> indexMaintainers = IndexMaintainer.deserialize(md, useProto);
-            final PhoenixTransactionContext txnContext = TransactionFactory.getTransactionProvider().getTransactionContext(txState);
-            byte[] clientVersionBytes = attributes.get(PhoenixIndexCodec.CLIENT_VERSION);
-            final int clientVersion = clientVersionBytes == null ? IndexMetaDataCache.UNKNOWN_CLIENT_VERSION : Bytes.toInt(clientVersionBytes);
+            byte[] clientVersionBytes = attributes.get(BaseScannerRegionObserver.CLIENT_VERSION);
+            final int clientVersion = clientVersionBytes == null ? ScanUtil.UNKNOWN_CLIENT_VERSION : Bytes.toInt(clientVersionBytes);
+            final PhoenixTransactionContext txnContext = TransactionFactory.getTransactionContext(txState, clientVersion);
             return new IndexMetaDataCache() {
 
                 @Override

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
index ba6a08f..cc7221e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
@@ -42,7 +42,6 @@ import org.apache.phoenix.cache.TenantCache;
 import org.apache.phoenix.coprocessor.BaseRegionScanner;
 import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
 import org.apache.phoenix.coprocessor.HashJoinRegionScanner;
-import org.apache.phoenix.execute.MutationState;
 import org.apache.phoenix.execute.TupleProjector;
 import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.expression.KeyValueColumnExpression;
@@ -63,6 +62,7 @@ import org.apache.phoenix.schema.tuple.ResultTuple;
 import org.apache.phoenix.schema.tuple.Tuple;
 import org.apache.phoenix.schema.types.PInteger;
 import org.apache.phoenix.transaction.PhoenixTransactionContext;
+import org.apache.phoenix.transaction.TransactionFactory;
 import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.ScanUtil;
@@ -128,12 +128,13 @@ public class NonAggregateRegionScannerFactory extends RegionScannerFactory {
       if (localIndexBytes == null) {
         localIndexBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD);
       }
+      int clientVersion = ScanUtil.getClientVersion(scan);
       List<IndexMaintainer> indexMaintainers =
           localIndexBytes == null ? null : IndexMaintainer.deserialize(localIndexBytes, useProto);
       indexMaintainer = indexMaintainers.get(0);
       viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan);
       byte[] txState = scan.getAttribute(BaseScannerRegionObserver.TX_STATE);
-      tx = MutationState.decodeTransaction(txState);
+      tx = TransactionFactory.getTransactionContext(txState, clientVersion);
     }
 
     final TupleProjector p = TupleProjector.deserializeProjectorFromScan(scan);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index 9caf7fb..add0628 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -47,6 +47,7 @@ import org.apache.phoenix.expression.function.SQLIndexTypeFunction;
 import org.apache.phoenix.expression.function.SQLTableTypeFunction;
 import org.apache.phoenix.expression.function.SQLViewTypeFunction;
 import org.apache.phoenix.expression.function.SqlTypeNameFunction;
+import org.apache.phoenix.expression.function.TransactionProviderNameFunction;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 import org.apache.phoenix.hbase.index.util.VersionUtil;
 import org.apache.phoenix.iterate.DelegateResultIterator;
@@ -297,6 +298,9 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
     public static final String TRANSACTIONAL = "TRANSACTIONAL";
     public static final byte[] TRANSACTIONAL_BYTES = Bytes.toBytes(TRANSACTIONAL);
 
+    public static final String TRANSACTION_PROVIDER = "TRANSACTION_PROVIDER";
+    public static final byte[] TRANSACTION_PROVIDER_BYTES = Bytes.toBytes(TRANSACTION_PROVIDER);
+
     public static final String UPDATE_CACHE_FREQUENCY = "UPDATE_CACHE_FREQUENCY";
     public static final byte[] UPDATE_CACHE_FREQUENCY_BYTES = Bytes.toBytes(UPDATE_CACHE_FREQUENCY);
 
@@ -1133,9 +1137,10 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
                     VIEW_STATEMENT + "," +
                     SQLViewTypeFunction.NAME + "(" + VIEW_TYPE + ") AS " + VIEW_TYPE + "," +
                     SQLIndexTypeFunction.NAME + "(" + INDEX_TYPE + ") AS " + INDEX_TYPE + "," +
-                    TRANSACTIONAL + "," +
+                    TRANSACTION_PROVIDER + " IS NOT NULL AS " + TRANSACTIONAL + "," +
                     IS_NAMESPACE_MAPPED + "," +
-                    GUIDE_POSTS_WIDTH +
+                    GUIDE_POSTS_WIDTH + "," +
+                    TransactionProviderNameFunction.NAME + "(" + TRANSACTION_PROVIDER + ") AS TRANSACTION_PROVIDER" +
                     " from " + SYSTEM_CATALOG + " " + SYSTEM_CATALOG_ALIAS +
                     " where " + COLUMN_NAME + " is null" +
                     " and " + COLUMN_FAMILY + " is null" +
@@ -1175,7 +1180,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
                     "'' " + INDEX_TYPE + "," +
                     "CAST(null AS BOOLEAN) " + TRANSACTIONAL + "," +
                     "CAST(null AS BOOLEAN) " + IS_NAMESPACE_MAPPED + "," +
-                    "CAST(null AS BIGINT) " + GUIDE_POSTS_WIDTH + "\n");
+                    "CAST(null AS BIGINT) " + GUIDE_POSTS_WIDTH + "," +
+                    "CAST(null AS VARCHAR) " + TRANSACTION_PROVIDER + "\n");
             buf.append(
                     " from " + SYSTEM_SEQUENCE + "\n");
             StringBuilder whereClause = new StringBuilder();

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index f526419..25b9fb0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -405,7 +405,7 @@ public class PhoenixStatement implements Statement, SQLCloseable {
                                 MutationState state = connection.getMutationState();
                                 MutationPlan plan = stmt.compilePlan(PhoenixStatement.this, Sequence.ValueOp.VALIDATE_SEQUENCE);
                                 if (plan.getTargetRef() != null && plan.getTargetRef().getTable() != null && plan.getTargetRef().getTable().isTransactional()) {
-                                    state.startTransaction();
+                                    state.startTransaction(plan.getTargetRef().getTable().getTransactionProvider());
                                 }
                                 Iterator<TableRef> tableRefs = plan.getSourceRefs().iterator();
                                 state.sendUncommitted(tableRefs);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexPartialBuildMapper.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexPartialBuildMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexPartialBuildMapper.java
index 7c154f0..f4ecac2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexPartialBuildMapper.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexPartialBuildMapper.java
@@ -116,7 +116,7 @@ public class PhoenixIndexPartialBuildMapper extends TableMapper<ImmutableBytesWr
                         put = new Put(CellUtil.cloneRow(cell));
                         put.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
                         put.setAttribute(PhoenixIndexCodec.INDEX_PROTO_MD, attribValue);
-                        put.setAttribute(PhoenixIndexCodec.CLIENT_VERSION, clientVersion);
+                        put.setAttribute(BaseScannerRegionObserver.CLIENT_VERSION, clientVersion);
                         put.setAttribute(BaseScannerRegionObserver.REPLAY_WRITES, BaseScannerRegionObserver.REPLAY_ONLY_INDEX_WRITES);
                         mutations.add(put);
                     }
@@ -126,7 +126,7 @@ public class PhoenixIndexPartialBuildMapper extends TableMapper<ImmutableBytesWr
                         del = new Delete(CellUtil.cloneRow(cell));
                         del.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
                         del.setAttribute(PhoenixIndexCodec.INDEX_PROTO_MD, attribValue);
-                        del.setAttribute(PhoenixIndexCodec.CLIENT_VERSION, clientVersion);
+                        del.setAttribute(BaseScannerRegionObserver.CLIENT_VERSION, clientVersion);
                         del.setAttribute(BaseScannerRegionObserver.REPLAY_WRITES, BaseScannerRegionObserver.REPLAY_ONLY_INDEX_WRITES);
                         mutations.add(del);
                     }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index 0b72ada..0598a0a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -48,6 +48,8 @@ import org.apache.phoenix.schema.SequenceAllocation;
 import org.apache.phoenix.schema.SequenceKey;
 import org.apache.phoenix.schema.stats.GuidePostsInfo;
 import org.apache.phoenix.schema.stats.GuidePostsKey;
+import org.apache.phoenix.transaction.PhoenixTransactionClient;
+import org.apache.phoenix.transaction.TransactionFactory;
 
 
 public interface ConnectionQueryServices extends QueryServices, MetaDataMutated {
@@ -152,4 +154,6 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
     public User getUser();
 
     public QueryLoggerDisruptor getQueryDisruptor();
+    
+    public PhoenixTransactionClient initTransactionClient(TransactionFactory.Provider provider);
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 6627a84..914f62a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -116,6 +116,7 @@ import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.coprocessor.Batch;
 import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
 import org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory;
@@ -138,10 +139,10 @@ import org.apache.phoenix.coprocessor.MetaDataProtocol;
 import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
 import org.apache.phoenix.coprocessor.MetaDataProtocol.MutationCode;
 import org.apache.phoenix.coprocessor.MetaDataRegionObserver;
-import org.apache.phoenix.coprocessor.PhoenixTransactionalProcessor;
 import org.apache.phoenix.coprocessor.ScanRegionObserver;
 import org.apache.phoenix.coprocessor.SequenceRegionObserver;
 import org.apache.phoenix.coprocessor.ServerCachingEndpointImpl;
+import org.apache.phoenix.coprocessor.TephraTransactionalProcessor;
 import org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver;
 import org.apache.phoenix.coprocessor.generated.MetaDataProtos;
 import org.apache.phoenix.coprocessor.generated.MetaDataProtos.AddColumnRequest;
@@ -226,8 +227,10 @@ import org.apache.phoenix.schema.types.PTinyint;
 import org.apache.phoenix.schema.types.PUnsignedTinyint;
 import org.apache.phoenix.schema.types.PVarbinary;
 import org.apache.phoenix.schema.types.PVarchar;
+import org.apache.phoenix.transaction.PhoenixTransactionClient;
 import org.apache.phoenix.transaction.PhoenixTransactionContext;
 import org.apache.phoenix.transaction.TransactionFactory;
+import org.apache.phoenix.transaction.TransactionFactory.Provider;
 import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.Closeables;
 import org.apache.phoenix.util.ConfigUtil;
@@ -264,7 +267,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     private static final int DEFAULT_OUT_OF_ORDER_MUTATIONS_WAIT_TIME_MS = 1000;
     private static final int TTL_FOR_MUTEX = 15 * 60; // 15min 
     protected final Configuration config;
-    private final ConnectionInfo connectionInfo;
+    protected final ConnectionInfo connectionInfo;
     // Copy of config.getProps(), but read-only to prevent synchronization that we
     // don't need.
     private final ReadOnlyProps props;
@@ -306,6 +309,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     // List of queues instead of a single queue to provide reduced contention via lock striping
     private final List<LinkedBlockingQueue<WeakReference<PhoenixConnection>>> connectionQueues;
     private ScheduledExecutorService renewLeaseExecutor;
+    private PhoenixTransactionClient[] txClients = new PhoenixTransactionClient[TransactionFactory.Provider.values().length];;
     /*
      * We can have multiple instances of ConnectionQueryServices. By making the thread factory
      * static, renew lease thread names will be unique across them.
@@ -410,23 +414,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
 
     }
 
-    private void initTxServiceClient() {
-        txZKClientService = TransactionFactory.getTransactionProvider().getTransactionContext().setTransactionClient(config, props, connectionInfo);
-    }
-
     private void openConnection() throws SQLException {
         try {
-            boolean transactionsEnabled = props.getBoolean(
-                    QueryServices.TRANSACTIONS_ENABLED,
-                    QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED);
             this.connection = HBaseFactoryProvider.getHConnectionFactory().createConnection(this.config);
             GLOBAL_HCONNECTIONS_COUNTER.increment();
             logger.info("HConnection established. Stacktrace for informational purposes: " + connection + " " +  LogUtil.getCallerStackTrace());
-            // only initialize the tx service client if needed and if we succeeded in getting a connection
-            // to HBase
-            if (transactionsEnabled) {
-                initTxServiceClient();
-            }
         } catch (IOException e) {
             throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ESTABLISH_CONNECTION)
             .setRootCause(e).build().buildException();
@@ -517,7 +509,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                             renewLeaseExecutor.shutdownNow();
                         }
                         // shut down the tx client service if we created one to support transactions
-                        if (this.txZKClientService != null) this.txZKClientService.stopAndWait();
+                        for (PhoenixTransactionClient client : txClients) {
+                            if (client != null) {
+                                client.close();
+                            }
+                        }
                     }
                 } catch (IOException e) {
                     if (sqlE == null) {
@@ -858,9 +854,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
             if (!descriptor.hasCoprocessor(ServerCachingEndpointImpl.class.getName())) {
                 descriptor.addCoprocessor(ServerCachingEndpointImpl.class.getName(), null, priority, null);
             }
+            // For ALTER TABLE
+            boolean nonTxToTx = Boolean.TRUE.equals(tableProps.get(PhoenixTransactionContext.READ_NON_TX_DATA));
             boolean isTransactional =
-                    Boolean.TRUE.equals(tableProps.get(TableProperty.TRANSACTIONAL.name())) ||
-                    Boolean.TRUE.equals(tableProps.get(PhoenixTransactionContext.READ_NON_TX_DATA)); // For ALTER TABLE
+                    Boolean.TRUE.equals(tableProps.get(TableProperty.TRANSACTIONAL.name())) || nonTxToTx;
             // TODO: better encapsulation for this
             // Since indexes can't have indexes, don't install our indexing coprocessor for indexes.
             // Also don't install on the SYSTEM.CATALOG and SYSTEM.STATS table because we use
@@ -923,13 +920,22 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
             }
 
             if (isTransactional) {
-                if (!descriptor.hasCoprocessor(PhoenixTransactionalProcessor.class.getName())) {
-                    descriptor.addCoprocessor(PhoenixTransactionalProcessor.class.getName(), null, priority - 10, null);
+                TransactionFactory.Provider provider = (TransactionFactory.Provider)TableProperty.TRANSACTION_PROVIDER.getValue(tableProps);
+                if (provider == null) {
+                    String providerValue = this.props.get(QueryServices.DEFAULT_TRANSACTION_PROVIDER_ATTRIB, QueryServicesOptions.DEFAULT_TRANSACTION_PROVIDER);
+                    provider = (TransactionFactory.Provider)TableProperty.TRANSACTION_PROVIDER.getValue(providerValue);
+                }
+                Class<? extends RegionObserver> coprocessorClass = provider.getTransactionProvider().getCoprocessor();
+                if (!descriptor.hasCoprocessor(coprocessorClass.getName())) {
+                    descriptor.addCoprocessor(coprocessorClass.getName(), null, priority - 10, null);
                 }
             } else {
-                // If exception on alter table to transition back to non transactional
-                if (descriptor.hasCoprocessor(PhoenixTransactionalProcessor.class.getName())) {
-                    descriptor.removeCoprocessor(PhoenixTransactionalProcessor.class.getName());
+                // Remove all potential transactional coprocessors
+                for (TransactionFactory.Provider provider : TransactionFactory.Provider.values()) {
+                    Class<? extends RegionObserver> coprocessorClass = provider.getTransactionProvider().getCoprocessor();
+                    if (coprocessorClass != null && descriptor.hasCoprocessor(coprocessorClass.getName())) {
+                        descriptor.removeCoprocessor(coprocessorClass.getName());
+                    }
                 }
             }
         } catch (IOException e) {
@@ -1126,7 +1132,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                 } else {
                     // If we think we're creating a non transactional table when it's already
                     // transactional, don't allow.
-                    if (existingDesc.hasCoprocessor(PhoenixTransactionalProcessor.class.getName())) {
+                    if (existingDesc.hasCoprocessor(TephraTransactionalProcessor.class.getName())) {
                         throw new SQLExceptionInfo.Builder(SQLExceptionCode.TX_MAY_NOT_SWITCH_TO_NON_TX)
                         .setSchemaName(SchemaUtil.getSchemaNameFromFullName(physicalTableName))
                         .setTableName(SchemaUtil.getTableNameFromFullName(physicalTableName)).build().buildException();
@@ -2895,6 +2901,14 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                                 + PBoolean.INSTANCE.getSqlTypeName());
                     addParentToChildLinks(metaConnection);
                 }
+                if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0) {
+                    metaConnection = addColumnsIfNotExists(
+                        metaConnection,
+                        PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+                        MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0,
+                        PhoenixDatabaseMetaData.TRANSACTION_PROVIDER + " "
+                                + PTinyint.INSTANCE.getSqlTypeName());
+                }
             }
 
 
@@ -4515,4 +4529,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     public QueryLoggerDisruptor getQueryDisruptor() {
         return this.queryDisruptor;
     }
+
+    @Override
+    public synchronized PhoenixTransactionClient initTransactionClient(Provider provider) {
+        PhoenixTransactionClient client = txClients[provider.ordinal()];
+        if (client == null) {
+            client = txClients[provider.ordinal()] = provider.getTransactionProvider().getTransactionClient(config, connectionInfo);
+        }
+        return client;
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index ad354d1..41df7b0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -82,7 +82,8 @@ import org.apache.phoenix.schema.TableAlreadyExistsException;
 import org.apache.phoenix.schema.TableNotFoundException;
 import org.apache.phoenix.schema.stats.GuidePostsInfo;
 import org.apache.phoenix.schema.stats.GuidePostsKey;
-import org.apache.phoenix.transaction.TransactionFactory;
+import org.apache.phoenix.transaction.PhoenixTransactionClient;
+import org.apache.phoenix.transaction.TransactionFactory.Provider;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.JDBCUtil;
 import org.apache.phoenix.util.MetaDataUtil;
@@ -141,7 +142,6 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
         // Without making a copy of the configuration we cons up, we lose some of our properties
         // on the server side during testing.
         this.config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration(config);
-        TransactionFactory.getTransactionProvider().getTransactionContext().setInMemoryTransactionClient(config);
         this.guidePostsCache = new GuidePostsCache(this, config);
     }
 
@@ -682,4 +682,9 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
     public QueryLoggerDisruptor getQueryDisruptor() {
         return null;
     }
+    
+    @Override
+    public PhoenixTransactionClient initTransactionClient(Provider provider) {
+        return null; // Client is not necessary
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index f5c8a59..e393210 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -49,6 +49,8 @@ import org.apache.phoenix.schema.SequenceAllocation;
 import org.apache.phoenix.schema.SequenceKey;
 import org.apache.phoenix.schema.stats.GuidePostsInfo;
 import org.apache.phoenix.schema.stats.GuidePostsKey;
+import org.apache.phoenix.transaction.PhoenixTransactionClient;
+import org.apache.phoenix.transaction.TransactionFactory.Provider;
 
 
 public class DelegateConnectionQueryServices extends DelegateQueryServices implements ConnectionQueryServices {
@@ -364,5 +366,8 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
         return getDelegate().getQueryDisruptor();
     }
     
-    
+    @Override
+    public PhoenixTransactionClient initTransactionClient(Provider provider) {
+        return getDelegate().initTransactionClient(provider);
+    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index ae12e01..d181fc8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -100,6 +100,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTION_PROVIDER;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE;
@@ -339,6 +340,7 @@ public interface QueryConstants {
             ENCODING_SCHEME + " TINYINT, " +
             COLUMN_QUALIFIER_COUNTER + " INTEGER, " +
             USE_STATS_FOR_PARALLELIZATION + " BOOLEAN, " +
+            TRANSACTION_PROVIDER + " TINYINT, " +
             "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ","
             + TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" +
             HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 21f043c..29d18d9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -207,6 +207,7 @@ public interface QueryServices extends SQLCloseable {
     public static final String DEFAULT_KEEP_DELETED_CELLS_ATTRIB = "phoenix.table.default.keep.deleted.cells";
     public static final String DEFAULT_STORE_NULLS_ATTRIB = "phoenix.table.default.store.nulls";
     public static final String DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB = "phoenix.table.istransactional.default";
+    public static final String DEFAULT_TRANSACTION_PROVIDER_ATTRIB = "phoenix.table.transaction.provider.default";
     public static final String GLOBAL_METRICS_ENABLED = "phoenix.query.global.metrics.enabled";
     
     // Transaction related configs

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 58c9812..70ac11b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -114,6 +114,7 @@ import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.PTableRefFactory;
 import org.apache.phoenix.trace.util.Tracing;
+import org.apache.phoenix.transaction.TransactionFactory;
 import org.apache.phoenix.util.DateUtil;
 import org.apache.phoenix.util.ReadOnlyProps;
 
@@ -263,6 +264,7 @@ public class QueryServicesOptions {
     // We'll also need this for transactions to work correctly
     public static final boolean DEFAULT_AUTO_COMMIT = false;
     public static final boolean DEFAULT_TABLE_ISTRANSACTIONAL = false;
+    public static final String DEFAULT_TRANSACTION_PROVIDER = TransactionFactory.Provider.getDefault().name();
     public static final boolean DEFAULT_TRANSACTIONS_ENABLED = false;
     public static final boolean DEFAULT_IS_GLOBAL_METRICS_ENABLED = true;
 
@@ -554,22 +556,22 @@ public class QueryServicesOptions {
         return set(GROUPBY_SPILL_FILES_ATTRIB, num);
     }
 
-    private QueryServicesOptions set(String name, boolean value) {
+    QueryServicesOptions set(String name, boolean value) {
         config.set(name, Boolean.toString(value));
         return this;
     }
 
-    private QueryServicesOptions set(String name, int value) {
+    QueryServicesOptions set(String name, int value) {
         config.set(name, Integer.toString(value));
         return this;
     }
 
-    private QueryServicesOptions set(String name, String value) {
+    QueryServicesOptions set(String name, String value) {
         config.set(name, value);
         return this;
     }
 
-    private QueryServicesOptions set(String name, long value) {
+    QueryServicesOptions set(String name, long value) {
         config.set(name, Long.toString(value));
         return this;
     }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java
index 8f15c5e..d1b8f1e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
 import org.apache.phoenix.index.IndexMaintainer;
 import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.transaction.TransactionFactory;
 
 public class DelegateTable implements PTable {
     @Override
@@ -237,7 +238,12 @@ public class DelegateTable implements PTable {
     }
 
     @Override
-    public boolean isTransactional() {
+    public TransactionFactory.Provider getTransactionProvider() {
+        return delegate.getTransactionProvider();
+    }
+
+    @Override
+    public final boolean isTransactional() {
         return delegate.isTransactional();
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index d252879..3c7a837 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -83,6 +83,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTION_PROVIDER;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION;
@@ -222,6 +223,9 @@ import org.apache.phoenix.schema.types.PUnsignedLong;
 import org.apache.phoenix.schema.types.PVarbinary;
 import org.apache.phoenix.schema.types.PVarchar;
 import org.apache.phoenix.transaction.PhoenixTransactionContext;
+import org.apache.phoenix.transaction.PhoenixTransactionProvider;
+import org.apache.phoenix.transaction.TransactionFactory;
+import org.apache.phoenix.transaction.TransactionFactory.Provider;
 import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.CursorUtil;
 import org.apache.phoenix.util.EncodedColumnsUtil;
@@ -280,7 +284,7 @@ public class MetaDataClient {
                     INDEX_TYPE + "," +
                     STORE_NULLS + "," +
                     BASE_COLUMN_COUNT + "," +
-                    TRANSACTIONAL + "," +
+                    TRANSACTION_PROVIDER + "," +
                     UPDATE_CACHE_FREQUENCY + "," +
                     IS_NAMESPACE_MAPPED + "," +
                     AUTO_PARTITION_SEQ +  "," +
@@ -572,14 +576,11 @@ public class MetaDataClient {
         } catch (TableNotFoundException e) {
         }
 
-        boolean defaultTransactional = connection.getQueryServices().getProps().getBoolean(
-                QueryServices.DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB,
-                QueryServicesOptions.DEFAULT_TRANSACTIONAL);
         // start a txn if all table are transactional by default or if we found the table in the cache and it is transactional
         // TODO if system tables become transactional remove the check
-        boolean isTransactional = defaultTransactional || (table!=null && table.isTransactional());
-        if (!systemTable && isTransactional && !connection.getMutationState().isTransactionStarted()) {
-            connection.getMutationState().startTransaction();
+        boolean isTransactional = (table!=null && table.isTransactional());
+        if (isTransactional) {
+            connection.getMutationState().startTransaction(table.getTransactionProvider());
         }
         resolvedTimestamp = resolvedTimestamp==null ? TransactionUtil.getResolvedTimestamp(connection, isTransactional, HConstants.LATEST_TIMESTAMP) : resolvedTimestamp;
         // Do not make rpc to getTable if
@@ -632,16 +633,20 @@ public class MetaDataClient {
                 result =
                         queryServices.getTable(tenantId, schemaBytes, tableBytes, tableTimestamp,
                             resolvedTimestamp);
-                // if the table was assumed to be transactional, but is actually not transactional
-                // then re-resolve as of the right timestamp (and vice versa)
-                if (table == null && result.getTable() != null
-                        && result.getTable().isTransactional() != isTransactional) {
-                    result =
-                            queryServices.getTable(tenantId, schemaBytes, tableBytes,
-                                tableTimestamp,
-                                TransactionUtil.getResolvedTimestamp(connection,
-                                    result.getTable().isTransactional(),
-                                    HConstants.LATEST_TIMESTAMP));
+                // if the table was assumed to be non transactional, but is actually transactional
+                // then re-resolve as of the right timestamp
+                if (result.getTable() != null
+                        && result.getTable().isTransactional()
+                        && !isTransactional) {
+                    long resolveTimestamp = TransactionUtil.getResolvedTimestamp(connection,
+                            result.getTable().isTransactional(),
+                            HConstants.LATEST_TIMESTAMP);
+                    // Reresolve if table timestamp is past timestamp as of which we should see data
+                    if (result.getTable().getTimeStamp() >= resolveTimestamp) {
+                        result =
+                                queryServices.getTable(tenantId, schemaBytes, tableBytes,
+                                    tableTimestamp, resolveTimestamp);
+                    }
                 }
 
                 if (SYSTEM_CATALOG_SCHEMA.equals(schemaName)) {
@@ -1236,8 +1241,8 @@ public class MetaDataClient {
             //view all the data belonging to the table
             PTable nonTxnLogicalTable = new DelegateTable(logicalTable) {
                 @Override
-                public boolean isTransactional() {
-                    return false;
+                public TransactionFactory.Provider getTransactionProvider() {
+                    return null;
                 }
             };
             TableRef tableRef = new TableRef(null, nonTxnLogicalTable, clientTimeStamp, false);
@@ -1861,7 +1866,7 @@ public class MetaDataClient {
             long clientTimeStamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
             boolean multiTenant = false;
             boolean storeNulls = false;
-            boolean transactional = (parent!= null) ? parent.isTransactional() : false;
+            TransactionFactory.Provider transactionProvider = (parent!= null) ? parent.getTransactionProvider() : null;
             Integer saltBucketNum = null;
             String defaultFamilyName = null;
             boolean isImmutableRows = false;
@@ -1877,7 +1882,7 @@ public class MetaDataClient {
             QualifierEncodingScheme encodingScheme = NON_ENCODED_QUALIFIERS;
             ImmutableStorageScheme immutableStorageScheme = ONE_CELL_PER_COLUMN;
             if (parent != null && tableType == PTableType.INDEX) {
-                timestamp = TransactionUtil.getTableTimestamp(connection, transactional);
+                timestamp = TransactionUtil.getTableTimestamp(connection, transactionProvider != null, transactionProvider);
                 storeNulls = parent.getStoreNulls();
                 isImmutableRows = parent.isImmutableRows();
                 isAppendOnlySchema = parent.isAppendOnlySchema();
@@ -2018,31 +2023,45 @@ public class MetaDataClient {
                 storeNulls = storeNullsProp;
             }
             Boolean transactionalProp = (Boolean) TableProperty.TRANSACTIONAL.getValue(tableProps);
-            if (transactionalProp != null && parent != null) {
+            TransactionFactory.Provider transactionProviderProp = (TransactionFactory.Provider) TableProperty.TRANSACTION_PROVIDER.getValue(tableProps);
+            if ((transactionalProp != null || transactionProviderProp != null) && parent != null) {
                 throw new SQLExceptionInfo.Builder(SQLExceptionCode.ONLY_TABLE_MAY_BE_DECLARED_TRANSACTIONAL)
                 .setSchemaName(schemaName).setTableName(tableName)
                 .build().buildException();
             }
             if (parent == null) {
-                if (transactionalProp == null) {
+                boolean transactional;
+                if (transactionProviderProp != null) {
+                    transactional = true;
+                } else if (transactionalProp == null) {
                     transactional = connection.getQueryServices().getProps().getBoolean(
                                     QueryServices.DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB,
                                     QueryServicesOptions.DEFAULT_TABLE_ISTRANSACTIONAL);
                 } else {
                     transactional = transactionalProp;
                 }
+                if (transactional) {
+                    if (transactionProviderProp == null) {
+                        transactionProvider = (TransactionFactory.Provider)TableProperty.TRANSACTION_PROVIDER.getValue(
+                                connection.getQueryServices().getProps().get(
+                                        QueryServices.DEFAULT_TRANSACTION_PROVIDER_ATTRIB,
+                                        QueryServicesOptions.DEFAULT_TRANSACTION_PROVIDER));
+                    } else {
+                        transactionProvider = transactionProviderProp;
+                    }
+                }
             }
             boolean transactionsEnabled = connection.getQueryServices().getProps().getBoolean(
                                             QueryServices.TRANSACTIONS_ENABLED,
                                             QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED);
             // can't create a transactional table if transactions are not enabled
-            if (!transactionsEnabled && transactional) {
+            if (!transactionsEnabled && transactionProvider != null) {
                 throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_CREATE_TXN_TABLE_IF_TXNS_DISABLED)
                 .setSchemaName(schemaName).setTableName(tableName)
                 .build().buildException();
             }
             // can't create a transactional table if it has a row timestamp column
-            if (pkConstraint.getNumColumnsWithRowTimestamp() > 0 && transactional) {
+            if (pkConstraint.getNumColumnsWithRowTimestamp() > 0 && transactionProvider != null) {
                 throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_CREATE_TXN_TABLE_WITH_ROW_TIMESTAMP)
                 .setSchemaName(schemaName).setTableName(tableName)
                 .build().buildException();
@@ -2050,8 +2069,9 @@ public class MetaDataClient {
 
             // Put potentially inferred value into tableProps as it's used by the createTable call below
             // to determine which coprocessors to install on the new table.
-            tableProps.put(PhoenixDatabaseMetaData.TRANSACTIONAL, transactional);
-            if (transactional) {
+            tableProps.put(PhoenixDatabaseMetaData.TRANSACTIONAL, transactionProvider != null);
+            if (transactionProvider != null) {
+                // TODO: for Omid
                 // If TTL set, use Tephra TTL property name instead
                 Object ttl = commonFamilyProps.remove(HColumnDescriptor.TTL);
                 if (ttl != null) {
@@ -2063,7 +2083,7 @@ public class MetaDataClient {
                     (Boolean) TableProperty.USE_STATS_FOR_PARALLELIZATION.getValue(tableProps);
 
             boolean sharedTable = statement.getTableType() == PTableType.VIEW || allocateIndexId;
-            if (transactional) {
+            if (transactionProvider != null) {
                 // Tephra uses an empty value cell as its delete marker, so we need to turn on
                 // storeNulls for transactional tables.
                 // If we use regular column delete markers (which is what non transactional tables
@@ -2098,7 +2118,7 @@ public class MetaDataClient {
                     }
                 }
             }
-            timestamp = timestamp==null ? TransactionUtil.getTableTimestamp(connection, transactional) : timestamp;
+            timestamp = timestamp==null ? TransactionUtil.getTableTimestamp(connection, transactionProvider != null, transactionProvider) : timestamp;
 
             // Delay this check as it is supported to have IMMUTABLE_ROWS and SALT_BUCKETS defined on views
             if (sharedTable) {
@@ -2481,7 +2501,7 @@ public class MetaDataClient {
                         Collections.<PTable>emptyList(), isImmutableRows,
                         Collections.<PName>emptyList(), defaultFamilyName == null ? null :
                                 PNameFactory.newName(defaultFamilyName), null,
-                        Boolean.TRUE.equals(disableWAL), false, false, null, null, indexType, true, false, 0, 0L, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, ONE_CELL_PER_COLUMN, NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, true);
+                        Boolean.TRUE.equals(disableWAL), false, false, null, null, indexType, true, null, 0, 0L, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, ONE_CELL_PER_COLUMN, NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, true);
                 connection.addTable(table, MetaDataProtocol.MIN_TABLE_TIMESTAMP);
             }
             
@@ -2621,7 +2641,11 @@ public class MetaDataClient {
             } else {
                 tableUpsert.setInt(19, BASE_TABLE_BASE_COLUMN_COUNT);
             }
-            tableUpsert.setBoolean(20, transactional);
+            if (transactionProvider == null) {
+                tableUpsert.setNull(20, Types.TINYINT);
+            } else {
+                tableUpsert.setByte(20, transactionProvider.getCode());
+            }
             tableUpsert.setLong(21, updateCacheFrequency);
             tableUpsert.setBoolean(22, isNamespaceMapped);
             if (autoPartitionSeq == null) {
@@ -2746,7 +2770,7 @@ public class MetaDataClient {
                         PTable.INITIAL_SEQ_NUM, pkName == null ? null : PNameFactory.newName(pkName), saltBucketNum, columns.values(),
                         parent == null ? null : parent.getSchemaName(), parent == null ? null : parent.getTableName(), Collections.<PTable>emptyList(), isImmutableRows,
                         physicalNames, defaultFamilyName == null ? null : PNameFactory.newName(defaultFamilyName), viewStatement, Boolean.TRUE.equals(disableWAL), multiTenant, storeNulls, viewType,
-                        result.getViewIndexId(), indexType, rowKeyOrderOptimizable, transactional, updateCacheFrequency, 0L, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, immutableStorageScheme, encodingScheme, cqCounterToBe, useStatsForParallelizationProp);
+                        result.getViewIndexId(), indexType, rowKeyOrderOptimizable, transactionProvider, updateCacheFrequency, 0L, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, immutableStorageScheme, encodingScheme, cqCounterToBe, useStatsForParallelizationProp);
                 result = new MetaDataMutationResult(code, result.getMutationTime(), table, true);
                 addTableToCache(result);
                 return table;
@@ -3251,8 +3275,8 @@ public class MetaDataClient {
                 changingPhoenixTableProperty = evaluateStmtProperties(metaProperties,metaPropertiesEvaluated,table,schemaName,tableName);
                 // If changing isImmutableRows to true or it's not being changed and is already true
                 boolean willBeImmutableRows = Boolean.TRUE.equals(metaPropertiesEvaluated.getIsImmutableRows()) || (metaPropertiesEvaluated.getIsImmutableRows() == null && table.isImmutableRows());
-
-                Long timeStamp = TransactionUtil.getTableTimestamp(connection, table.isTransactional() || metaProperties.getNonTxToTx());
+                boolean willBeTxnl = metaProperties.getNonTxToTx();
+                Long timeStamp = TransactionUtil.getTableTimestamp(connection, table.isTransactional() || willBeTxnl, metaPropertiesEvaluated.getTransactionProvider());
                 int numPkColumnsAdded = 0;
                 List<PColumn> columns = Lists.newArrayListWithExpectedSize(numCols);
                 Set<String> colFamiliesForPColumnsToBeAdded = new LinkedHashSet<>();
@@ -4247,6 +4271,8 @@ public class MetaDataClient {
                         metaProperties.setStoreNullsProp((Boolean)value);
                     } else if (propName.equals(TRANSACTIONAL)) {
                         metaProperties.setIsTransactionalProp((Boolean)value);
+                    } else if (propName.equals(TRANSACTION_PROVIDER)) {
+                        metaProperties.setTransactionProviderProp((TransactionFactory.Provider) value);
                     } else if (propName.equals(UPDATE_CACHE_FREQUENCY)) {
                         metaProperties.setUpdateCacheFrequencyProp((Long)value);
                     } else if (propName.equals(GUIDE_POSTS_WIDTH)) {
@@ -4369,6 +4395,22 @@ public class MetaDataClient {
                             .setSchemaName(schemaName).setTableName(tableName)
                             .build().buildException();
                 }
+                TransactionFactory.Provider provider = metaProperties.getTransactionProviderProp();
+                if (provider == null) {
+                    provider = (Provider)
+                            TableProperty.TRANSACTION_PROVIDER.getValue(
+                                    connection.getQueryServices().getProps().get(
+                                            QueryServices.DEFAULT_TRANSACTION_PROVIDER_ATTRIB,
+                                            QueryServicesOptions.DEFAULT_TRANSACTION_PROVIDER));
+                    metaPropertiesEvaluated.setTransactionProvider(provider);
+                }
+                if (provider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.ALTER_NONTX_TO_TX)) {
+                    throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_TABLE_FROM_NON_TXN_TO_TXNL)
+                        .setMessage(provider.name() + ". ")
+                        .setSchemaName(schemaName)
+                        .setTableName(tableName)
+                        .build().buildException();
+                }
                 changingPhoenixTableProperty = true;
                 metaProperties.setNonTxToTx(true);
             }
@@ -4381,6 +4423,7 @@ public class MetaDataClient {
         private Boolean multiTenantProp = null;
         private Boolean disableWALProp = null;
         private Boolean storeNullsProp = null;
+        private TransactionFactory.Provider transactionProviderProp = null;
         private Boolean isTransactionalProp = null;
         private Long updateCacheFrequencyProp = null;
         private Boolean appendOnlySchemaProp = null;
@@ -4421,6 +4464,14 @@ public class MetaDataClient {
             this.storeNullsProp = storeNullsProp;
         }
 
+        public TransactionFactory.Provider getTransactionProviderProp() {
+            return transactionProviderProp;
+        }
+
+        public void setTransactionProviderProp(TransactionFactory.Provider transactionProviderProp) {
+            this.transactionProviderProp = transactionProviderProp;
+        }
+
         public Boolean getIsTransactionalProp() {
             return isTransactionalProp;
         }
@@ -4490,6 +4541,7 @@ public class MetaDataClient {
         private Boolean storeNulls = null;
         private Boolean useStatsForParallelization = null;
         private Boolean isTransactional = null;
+        private TransactionFactory.Provider transactionProvider = null;
 
         public Boolean getIsImmutableRows() {
             return isImmutableRows;
@@ -4570,5 +4622,14 @@ public class MetaDataClient {
         public void setIsTransactional(Boolean isTransactional) {
             this.isTransactional = isTransactional;
         }
+        
+        public TransactionFactory.Provider getTransactionProvider() {
+            return transactionProvider;
+        }
+
+        public void setTransactionProvider(TransactionFactory.Provider transactionProvider) {
+            this.transactionProvider = transactionProvider;
+        }
+
     }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
index 7e186ad..af78612 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
@@ -40,6 +40,7 @@ import org.apache.phoenix.schema.types.PArrayDataTypeDecoder;
 import org.apache.phoenix.schema.types.PArrayDataTypeEncoder;
 import org.apache.phoenix.schema.types.PDataType;
 import org.apache.phoenix.schema.types.PVarbinary;
+import org.apache.phoenix.transaction.TransactionFactory;
 import org.apache.phoenix.util.TrustedByteArrayOutputStream;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -680,6 +681,7 @@ public interface PTable extends PMetaDataEntity {
     boolean isMultiTenant();
     boolean getStoreNulls();
     boolean isTransactional();
+    TransactionFactory.Provider getTransactionProvider();
 
     ViewType getViewType();
     String getViewStatement();


[28/50] [abbrv] phoenix git commit: PHOENIX-4601 Perform server-side retries if client version < 4.14

Posted by td...@apache.org.
PHOENIX-4601 Perform server-side retries if client version < 4.14


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d7afec21
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d7afec21
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d7afec21

Branch: refs/heads/system-catalog
Commit: d7afec21c500ccd7f5334e9c0958c40fcc95df14
Parents: ccd41de
Author: Vincent Poon <vi...@apache.org>
Authored: Thu Apr 19 17:07:56 2018 -0700
Committer: Vincent Poon <vi...@apache.org>
Committed: Fri Apr 20 14:22:43 2018 -0700

----------------------------------------------------------------------
 .../hbase/index/write/IndexWriterUtils.java     | 36 ++++++++++++++++----
 .../write/ParallelWriterIndexCommitter.java     | 12 +++++--
 .../TrackingParallelWriterIndexCommitter.java   | 13 ++++---
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |  1 +
 4 files changed, 48 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/d7afec21/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
index 76d6800..0d3004f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
@@ -21,6 +21,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.phoenix.hbase.index.table.HTableFactory;
 import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
 import org.apache.phoenix.util.PropertiesUtil;
@@ -58,15 +59,19 @@ public class IndexWriterUtils {
    public static final String INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY = "phoenix.index.writes.threads.max";
    public static final String HTABLE_KEEP_ALIVE_KEY = "hbase.htable.threads.keepalivetime";
 
+   @Deprecated
    public static final String INDEX_WRITER_RPC_RETRIES_NUMBER = "phoenix.index.writes.rpc.retries.number";
-    /**
-     * Retry server-server index write rpc only once, and let the client retry the data write
-     * instead to avoid typing up the handler
-     */
-   // note in HBase 2+, numTries = numRetries + 1
-   // in prior versions, numTries = numRetries
-   public static final int DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER = 1;
+   /**
+    * Based on the logic in HBase's AsyncProcess, a default of 11 retries with a pause of 100ms
+    * approximates 48 sec total retry time (factoring in backoffs).  The total time should be less
+    * than HBase's rpc timeout (default of 60 sec) or else the client will retry before receiving
+    * the response
+    */
+   @Deprecated
+   public static final int DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER = 11;
+   @Deprecated
    public static final String INDEX_WRITER_RPC_PAUSE = "phoenix.index.writes.rpc.pause";
+   @Deprecated
    public static final int DEFAULT_INDEX_WRITER_RPC_PAUSE = 100;
 
   private IndexWriterUtils() {
@@ -76,12 +81,29 @@ public class IndexWriterUtils {
     public static HTableFactory getDefaultDelegateHTableFactory(CoprocessorEnvironment env) {
         // create a simple delegate factory, setup the way we need
         Configuration conf = PropertiesUtil.cloneConfig(env.getConfiguration());
+        setHTableThreads(conf);
+        return ServerUtil.getDelegateHTableFactory(env, conf);
+    }
+
+    private static void setHTableThreads(Configuration conf) {
         // set the number of threads allowed per table.
         int htableThreads =
                 conf.getInt(IndexWriterUtils.INDEX_WRITER_PER_TABLE_THREADS_CONF_KEY,
                     IndexWriterUtils.DEFAULT_NUM_PER_TABLE_THREADS);
         LOG.trace("Creating HTableFactory with " + htableThreads + " threads for each HTable.");
         IndexManagementUtil.setIfNotSet(conf, HTABLE_THREAD_KEY, htableThreads);
+    }
+
+    /**
+     * Retry server-server index write rpc only once, and let the client retry the data write
+     * instead to avoid tying up the handler
+     */
+    public static HTableFactory getNoRetriesHTableFactory(CoprocessorEnvironment env) {
+        Configuration conf = PropertiesUtil.cloneConfig(env.getConfiguration());
+        setHTableThreads(conf);
+        // note in HBase 2+, numTries = numRetries + 1
+        // in prior versions, numTries = numRetries
+        conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
         return ServerUtil.getDelegateHTableFactory(env, conf);
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d7afec21/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
index aba2678..cd196d3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
@@ -36,6 +36,7 @@ import org.apache.phoenix.hbase.index.table.HTableFactory;
 import org.apache.phoenix.hbase.index.table.HTableInterfaceReference;
 import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
 import org.apache.phoenix.index.PhoenixIndexFailurePolicy;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.util.IndexUtil;
 
 import com.google.common.collect.Multimap;
@@ -57,7 +58,8 @@ public class ParallelWriterIndexCommitter implements IndexCommitter {
     public static final String INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY = "index.writer.threads.keepalivetime";
     private static final Log LOG = LogFactory.getLog(ParallelWriterIndexCommitter.class);
 
-    private HTableFactory factory;
+    private HTableFactory retryingFactory;
+    private HTableFactory noRetriesfactory;
     private Stoppable stopped;
     private QuickFailingTaskRunner pool;
     private KeyValueBuilder kvBuilder;
@@ -80,6 +82,7 @@ public class ParallelWriterIndexCommitter implements IndexCommitter {
                                 DEFAULT_CONCURRENT_INDEX_WRITER_THREADS).setCoreTimeout(
                                 INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY), env), env.getRegionServerServices(), parent, env);
         this.kvBuilder = KeyValueBuilder.get(env.getHBaseVersion());
+        this.noRetriesfactory = IndexWriterUtils.getNoRetriesHTableFactory(env);
     }
 
     /**
@@ -88,7 +91,7 @@ public class ParallelWriterIndexCommitter implements IndexCommitter {
      * Exposed for TESTING
      */
     void setup(HTableFactory factory, ExecutorService pool, Abortable abortable, Stoppable stop, RegionCoprocessorEnvironment env) {
-        this.factory = factory;
+        this.retryingFactory = factory;
         this.pool = new QuickFailingTaskRunner(pool);
         this.stopped = stop;
     }
@@ -162,6 +165,8 @@ public class ParallelWriterIndexCommitter implements IndexCommitter {
                                 }
                             }
                         }
+                     // if the client can retry index writes, then we don't need to retry here
+                        HTableFactory factory = clientVersion < PhoenixDatabaseMetaData.MIN_CLIENT_RETRY_INDEX_WRITES ? retryingFactory : noRetriesfactory;
                         table = factory.getTable(tableReference.get());
                         throwFailureIfDone();
                         table.batch(mutations);
@@ -226,7 +231,8 @@ public class ParallelWriterIndexCommitter implements IndexCommitter {
     public void stop(String why) {
         LOG.info("Shutting down " + this.getClass().getSimpleName() + " because " + why);
         this.pool.stop(why);
-        this.factory.shutdown();
+        this.retryingFactory.shutdown();
+        this.noRetriesfactory.shutdown();
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d7afec21/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
index 4dbad63..1727fff 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
@@ -41,6 +41,7 @@ import org.apache.phoenix.hbase.index.table.HTableFactory;
 import org.apache.phoenix.hbase.index.table.HTableInterfaceReference;
 import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
 import org.apache.phoenix.index.PhoenixIndexFailurePolicy;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.util.IndexUtil;
 
 import com.google.common.collect.Multimap;
@@ -74,7 +75,8 @@ public class TrackingParallelWriterIndexCommitter implements IndexCommitter {
     private static final String INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY = "index.writer.threads.keepalivetime";
 
     private TaskRunner pool;
-    private HTableFactory factory;
+    private HTableFactory retryingFactory;
+    private HTableFactory noRetriesFactory;
     private CapturingAbortable abortable;
     private Stoppable stopped;
     private RegionCoprocessorEnvironment env;
@@ -98,6 +100,7 @@ public class TrackingParallelWriterIndexCommitter implements IndexCommitter {
                                 DEFAULT_CONCURRENT_INDEX_WRITER_THREADS).setCoreTimeout(
                                 INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY), env), env.getRegionServerServices(), parent, env);
         this.kvBuilder = KeyValueBuilder.get(env.getHBaseVersion());
+        this.noRetriesFactory = IndexWriterUtils.getNoRetriesHTableFactory(env);
     }
 
     /**
@@ -108,7 +111,7 @@ public class TrackingParallelWriterIndexCommitter implements IndexCommitter {
     void setup(HTableFactory factory, ExecutorService pool, Abortable abortable, Stoppable stop,
             RegionCoprocessorEnvironment env) {
         this.pool = new WaitForCompletionTaskRunner(pool);
-        this.factory = factory;
+        this.retryingFactory = factory;
         this.abortable = new CapturingAbortable(abortable);
         this.stopped = stop;
         this.env = env;
@@ -175,7 +178,8 @@ public class TrackingParallelWriterIndexCommitter implements IndexCommitter {
                         if (LOG.isTraceEnabled()) {
                             LOG.trace("Writing index update:" + mutations + " to table: " + tableReference);
                         }
-
+                        // if the client can retry index writes, then we don't need to retry here
+                        HTableFactory factory = clientVersion < PhoenixDatabaseMetaData.MIN_CLIENT_RETRY_INDEX_WRITES ? retryingFactory : noRetriesFactory;
                         table = factory.getTable(tableReference.get());
                         throwFailureIfDone();
                         table.batch(mutations);
@@ -238,7 +242,8 @@ public class TrackingParallelWriterIndexCommitter implements IndexCommitter {
     public void stop(String why) {
         LOG.info("Shutting down " + this.getClass().getSimpleName());
         this.pool.stop(why);
-        this.factory.shutdown();
+        this.retryingFactory.shutdown();
+        this.noRetriesFactory.shutdown();
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d7afec21/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index add0628..d56628a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -327,6 +327,7 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
     public static final int MIN_NAMESPACE_MAPPED_PHOENIX_VERSION = VersionUtil.encodeVersion("4", "8", "0");
     public static final int MIN_PENDING_ACTIVE_INDEX = VersionUtil.encodeVersion("4", "12", "0");
     public static final int MIN_PENDING_DISABLE_INDEX = VersionUtil.encodeVersion("4", "14", "0");
+    public static final int MIN_CLIENT_RETRY_INDEX_WRITES = VersionUtil.encodeVersion("4", "14", "0");
     public static final int MIN_TX_CLIENT_SIDE_MAINTENANCE = VersionUtil.encodeVersion("4", "14", "0");
     
     // Version below which we should turn off essential column family.


[17/50] [abbrv] phoenix git commit: Revert "PHOENIX-4496 Fix RowValueConstructorIT and IndexMetadataIT"

Posted by td...@apache.org.
Revert "PHOENIX-4496 Fix RowValueConstructorIT and IndexMetadataIT"

This reverts commit 70c9c6bb2d2073bb40640a0a794032b166c4b63b.


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7b2c7e13
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7b2c7e13
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7b2c7e13

Branch: refs/heads/system-catalog
Commit: 7b2c7e135916d5a6087ec3e6c4b6c261779795ac
Parents: 70c9c6b
Author: James Taylor <jt...@salesforce.com>
Authored: Fri Apr 13 10:10:07 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Apr 13 10:10:07 2018 -0700

----------------------------------------------------------------------
 .../phoenix/hbase/index/scanner/ScannerBuilder.java       | 10 ----------
 1 file changed, 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/7b2c7e13/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
index 4c42fe4..ad09c0c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.FamilyFilter;
 import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
 import org.apache.hadoop.hbase.filter.FilterList;
 import org.apache.hadoop.hbase.filter.QualifierFilter;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -99,15 +98,6 @@ public class ScannerBuilder {
       }
       columnFilters.addFilter(columnFilter);
     }
-    
-    if(columns.isEmpty()){
-        columnFilters.addFilter(new FilterBase(){
-            @Override
-            public boolean filterAllRemaining() throws IOException {
-                return true;
-            }
-        });
-    }
     return columnFilters;
   }
 


[03/50] [abbrv] phoenix git commit: PHOENIX-2715 Query Log (Ankit Singhal)

Posted by td...@apache.org.
PHOENIX-2715 Query Log (Ankit Singhal)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/0bc03684
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/0bc03684
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/0bc03684

Branch: refs/heads/system-catalog
Commit: 0bc0368455ae03de2f3f2cabd72de06e5cb90cbb
Parents: 98a8ddd
Author: Ankit Singhal <an...@gmail.com>
Authored: Tue Apr 10 12:21:00 2018 +0530
Committer: Ankit Singhal <an...@gmail.com>
Committed: Tue Apr 10 12:21:00 2018 +0530

----------------------------------------------------------------------
 phoenix-core/pom.xml                            |   5 +
 .../end2end/QueryDatabaseMetaDataIT.java        |   4 +
 .../apache/phoenix/end2end/QueryLoggerIT.java   | 358 +++++++++++++++++++
 .../end2end/TenantSpecificTablesDDLIT.java      |   2 +
 .../phoenix/compile/StatementContext.java       |  10 +
 .../phoenix/coprocessor/MetaDataProtocol.java   |   5 +-
 .../phoenix/iterate/ScanningResultIterator.java |  18 +-
 .../apache/phoenix/jdbc/PhoenixConnection.java  |  17 +-
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |  18 +
 .../phoenix/jdbc/PhoenixPreparedStatement.java  |  11 +-
 .../apache/phoenix/jdbc/PhoenixResultSet.java   |  38 ++
 .../apache/phoenix/jdbc/PhoenixStatement.java   |  67 +++-
 .../java/org/apache/phoenix/log/LogLevel.java   |  22 ++
 .../java/org/apache/phoenix/log/LogWriter.java  |  51 +++
 .../log/QueryLogDetailsEventHandler.java        |  63 ++++
 .../org/apache/phoenix/log/QueryLogInfo.java    |  87 +++++
 .../org/apache/phoenix/log/QueryLogState.java   |  22 ++
 .../org/apache/phoenix/log/QueryLogger.java     | 145 ++++++++
 .../log/QueryLoggerDefaultExceptionHandler.java |  51 +++
 .../phoenix/log/QueryLoggerDisruptor.java       | 117 ++++++
 .../org/apache/phoenix/log/QueryLoggerUtil.java |  62 ++++
 .../org/apache/phoenix/log/RingBufferEvent.java |  93 +++++
 .../phoenix/log/RingBufferEventTranslator.java  |  53 +++
 .../org/apache/phoenix/log/TableLogWriter.java  | 125 +++++++
 .../phoenix/monitoring/ReadMetricQueue.java     |  44 ++-
 .../phoenix/monitoring/ScanMetricsHolder.java   |  48 ++-
 .../phoenix/query/ConnectionQueryServices.java  |   6 +
 .../query/ConnectionQueryServicesImpl.java      |  35 +-
 .../query/ConnectionlessQueryServicesImpl.java  |  18 +
 .../query/DelegateConnectionQueryServices.java  |  14 +
 .../apache/phoenix/query/QueryConstants.java    |  45 +++
 .../org/apache/phoenix/query/QueryServices.java |   4 +
 .../phoenix/query/QueryServicesOptions.java     |   9 +-
 pom.xml                                         |   6 +
 34 files changed, 1612 insertions(+), 61 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index a64d4c3..76ca9c9 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -480,5 +480,10 @@
       <artifactId>i18n-util</artifactId>
       <version>${i18n-util.version}</version>
     </dependency>
+       <dependency>
+        <groupId>com.lmax</groupId>
+        <artifactId>disruptor</artifactId>
+        <version>${disruptor.version}</version>
+      </dependency>
   </dependencies>
 </project>

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
index 7a2d3e1..c7c8ebf 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
@@ -166,6 +166,10 @@ public class QueryDatabaseMetaDataIT extends ParallelStatsDisabledIT {
             assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
             assertTrue(rs.next());
             assertEquals(SYSTEM_CATALOG_SCHEMA, rs.getString("TABLE_SCHEM"));
+            assertEquals(PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE, rs.getString("TABLE_NAME"));
+            assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
+            assertTrue(rs.next());
+            assertEquals(SYSTEM_CATALOG_SCHEMA, rs.getString("TABLE_SCHEM"));
             assertEquals(TYPE_SEQUENCE, rs.getString("TABLE_NAME"));
             assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
             assertTrue(rs.next());

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java
new file mode 100644
index 0000000..940ba6f
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java
@@ -0,0 +1,358 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BIND_PARAMETERS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLIENT_IP;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXCEPTION_TRACE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXPLAIN_PLAN;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GLOBAL_SCAN_DETAILS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NO_OF_RESULTS_ITERATED;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_ID;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_STATUS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCAN_METRICS_JSON;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_TIME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TOTAL_EXECUTION_TIME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USER;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.net.InetAddress;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.phoenix.compile.StatementContext;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDriver;
+import org.apache.phoenix.jdbc.PhoenixResultSet;
+import org.apache.phoenix.log.LogLevel;
+import org.apache.phoenix.log.QueryLogState;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+public class QueryLoggerIT extends BaseUniqueNamesOwnClusterIT {
+
+
+    @BeforeClass
+    public static void doSetup() throws Exception {
+        Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+        // Enable request metric collection at the driver level
+        props.put(QueryServices.COLLECT_REQUEST_LEVEL_METRICS, String.valueOf(true));
+        // disable renewing leases as this will force spooling to happen.
+        props.put(QueryServices.RENEW_LEASE_ENABLED, String.valueOf(false));
+        setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+        // need the non-test driver for some tests that check number of hconnections, etc.
+        DriverManager.registerDriver(PhoenixDriver.INSTANCE);
+    } 
+    
+
+    @Test
+    public void testDebugLogs() throws Exception {
+        String tableName = generateUniqueName();
+        createTableAndInsertValues(tableName, true);
+        Properties props= new Properties();
+        props.setProperty(QueryServices.LOG_LEVEL, LogLevel.DEBUG.name());
+        Connection conn = DriverManager.getConnection(getUrl(),props);
+        assertEquals(conn.unwrap(PhoenixConnection.class).getLogLevel(),LogLevel.DEBUG);
+        String query = "SELECT * FROM " + tableName;
+        ResultSet rs = conn.createStatement().executeQuery(query);
+        StatementContext context = ((PhoenixResultSet)rs).getContext();
+        String queryId = context.getQueryLogger().getQueryId();
+        while (rs.next()) {
+            rs.getString(1);
+            rs.getString(2);
+        }
+        ResultSet explainRS = conn.createStatement().executeQuery("Explain " + query);
+
+        String logQuery = "SELECT * FROM " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_LOG_TABLE + "\"";
+        rs = conn.createStatement().executeQuery(logQuery);
+        boolean foundQueryLog = false;
+        int delay = 5000;
+
+        // sleep for sometime to let query log committed
+        Thread.sleep(delay);
+        while (rs.next()) {
+            if (rs.getString(QUERY_ID).equals(queryId)) {
+                foundQueryLog = true;
+                assertEquals(rs.getString(BIND_PARAMETERS), null);
+                assertEquals(rs.getString(USER), System.getProperty("user.name"));
+                assertEquals(rs.getString(CLIENT_IP), InetAddress.getLocalHost().getHostAddress());
+                assertEquals(rs.getString(EXPLAIN_PLAN), QueryUtil.getExplainPlan(explainRS));
+                assertEquals(rs.getString(GLOBAL_SCAN_DETAILS), context.getScan().toJSON());
+                assertEquals(rs.getLong(NO_OF_RESULTS_ITERATED), 10);
+                assertEquals(rs.getString(QUERY), query);
+                assertEquals(rs.getString(QUERY_STATUS), QueryLogState.COMPLETED.toString());
+                assertTrue(System.currentTimeMillis() - rs.getTimestamp(START_TIME).getTime() > delay);
+                assertEquals(rs.getString(TENANT_ID), null);
+                assertTrue(rs.getString(TOTAL_EXECUTION_TIME) != null);
+                assertTrue(rs.getString(SCAN_METRICS_JSON).contains("scanMetrics"));
+                assertEquals(rs.getString(EXCEPTION_TRACE),null);
+            }else{
+                //confirm we are not logging system queries
+                assertFalse(rs.getString(QUERY).toString().contains(SYSTEM_CATALOG_SCHEMA));
+            }
+        }
+        assertTrue(foundQueryLog);
+        conn.close();
+    }
+    
+    @Test
+    public void testLogSampling() throws Exception {
+        String tableName = generateUniqueName();
+        createTableAndInsertValues(tableName, true);
+        Properties props= new Properties();
+        props.setProperty(QueryServices.LOG_LEVEL, LogLevel.DEBUG.name());
+        props.setProperty(QueryServices.LOG_SAMPLE_RATE, "0.5");
+        Connection conn = DriverManager.getConnection(getUrl(),props);
+        assertEquals(conn.unwrap(PhoenixConnection.class).getLogLevel(),LogLevel.DEBUG);
+        String query = "SELECT * FROM " + tableName;
+        int count=100;
+        for (int i = 0; i < count; i++) {
+            conn.createStatement().executeQuery(query);
+        }
+        
+        String logQuery = "SELECT * FROM " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_LOG_TABLE + "\"";
+        ResultSet rs = conn.createStatement().executeQuery(logQuery);
+        int delay = 5000;
+
+        // sleep for sometime to let query log committed
+        Thread.sleep(delay);
+        int logCount=0;
+        while (rs.next()) {
+            logCount++;
+        }
+        
+        //sampling rate is 0.5 , but with lesser count, uniformity of thread random may not be perfect, so taking 0.75 for comparison 
+        assertTrue(logCount != 0 && logCount < count * 0.75);
+        conn.close();
+    }
+    
+    @Test
+    public void testInfoLogs() throws Exception{
+        String tableName = generateUniqueName();
+        createTableAndInsertValues(tableName, true);
+        Properties props= new Properties();
+        props.setProperty(QueryServices.LOG_LEVEL, LogLevel.INFO.name());
+        Connection conn = DriverManager.getConnection(getUrl(),props);
+        assertEquals(conn.unwrap(PhoenixConnection.class).getLogLevel(),LogLevel.INFO);
+        String query = "SELECT * FROM " + tableName;
+        
+        ResultSet rs = conn.createStatement().executeQuery(query);
+        StatementContext context = ((PhoenixResultSet)rs).getContext();
+        String queryId = context.getQueryLogger().getQueryId();
+        while (rs.next()) {
+            rs.getString(1);
+            rs.getString(2);
+        }
+
+        String logQuery = "SELECT * FROM " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_LOG_TABLE + "\"";
+        rs = conn.createStatement().executeQuery(logQuery);
+        boolean foundQueryLog = false;
+        int delay = 5000;
+
+        // sleep for sometime to let query log committed
+        Thread.sleep(delay);
+        while (rs.next()) {
+            if (rs.getString(QUERY_ID).equals(queryId)) {
+                foundQueryLog = true;
+                assertEquals(rs.getString(USER), System.getProperty("user.name"));
+                assertEquals(rs.getString(CLIENT_IP), InetAddress.getLocalHost().getHostAddress());
+                assertEquals(rs.getString(EXPLAIN_PLAN), null);
+                assertEquals(rs.getString(GLOBAL_SCAN_DETAILS),null);
+                assertEquals(rs.getLong(NO_OF_RESULTS_ITERATED), 0);
+                assertEquals(rs.getString(QUERY), query);
+                assertEquals(rs.getString(QUERY_STATUS),null);
+                assertTrue(System.currentTimeMillis() - rs.getTimestamp(START_TIME).getTime() > delay);
+                assertEquals(rs.getString(TENANT_ID), null);
+                assertTrue(rs.getString(TOTAL_EXECUTION_TIME) == null);
+            }
+        }
+        assertTrue(foundQueryLog);
+        conn.close();
+    }
+    
+    @Test
+    public void testWithLoggingOFF() throws Exception{
+        String tableName = generateUniqueName();
+        createTableAndInsertValues(tableName, true);
+        Properties props= new Properties();
+        props.setProperty(QueryServices.LOG_LEVEL, LogLevel.OFF.name());
+        Connection conn = DriverManager.getConnection(getUrl(),props);
+        assertEquals(conn.unwrap(PhoenixConnection.class).getLogLevel(),LogLevel.OFF);
+        String query = "SELECT * FROM " + tableName;
+        
+        ResultSet rs = conn.createStatement().executeQuery(query);
+        StatementContext context = ((PhoenixResultSet)rs).getContext();
+        String queryId = context.getQueryLogger().getQueryId();
+        while (rs.next()) {
+            rs.getString(1);
+            rs.getString(2);
+        }
+
+        String logQuery = "SELECT * FROM " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_LOG_TABLE + "\"";
+        rs = conn.createStatement().executeQuery(logQuery);
+        boolean foundQueryLog = false;
+        int delay = 5000;
+
+        // sleep for sometime to let query log committed
+        Thread.sleep(delay);
+        while (rs.next()) {
+            if (rs.getString(QUERY_ID).equals(queryId)) {
+                foundQueryLog = true;
+            }
+        }
+        assertFalse(foundQueryLog);
+        conn.close();
+    }
+    
+
+    @Test
+    public void testPreparedStatementWithTrace() throws Exception{
+        testPreparedStatement(LogLevel.TRACE);   
+    }
+    
+    @Test
+    public void testPreparedStatementWithDebug() throws Exception{
+        testPreparedStatement(LogLevel.DEBUG);
+    }
+            
+    private void testPreparedStatement(LogLevel loglevel) throws Exception{
+        String tableName = generateUniqueName();
+        createTableAndInsertValues(tableName, true);
+        Properties props= new Properties();
+        props.setProperty(QueryServices.LOG_LEVEL, loglevel.name());
+        Connection conn = DriverManager.getConnection(getUrl(),props);
+        assertEquals(conn.unwrap(PhoenixConnection.class).getLogLevel(),loglevel);
+        
+        String query = "SELECT * FROM " + tableName +" where V = ?";
+        
+        PreparedStatement pstmt = conn.prepareStatement(query);
+        pstmt.setString(1, "value5");
+        ResultSet rs = pstmt.executeQuery();
+        StatementContext context = ((PhoenixResultSet)rs).getContext();
+        String queryId = context.getQueryLogger().getQueryId();
+        while (rs.next()) {
+            rs.getString(1);
+            rs.getString(2);
+        }
+        ResultSet explainRS = conn.createStatement()
+                .executeQuery("Explain " + "SELECT * FROM " + tableName + " where V = 'value5'");
+        String logQuery = "SELECT * FROM " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_LOG_TABLE + "\"";
+        rs = conn.createStatement().executeQuery(logQuery);
+        boolean foundQueryLog = false;
+        int delay = 5000;
+        
+        // sleep for sometime to let query log committed
+        Thread.sleep(delay);
+        while (rs.next()) {
+            if (rs.getString(QUERY_ID).equals(queryId)) {
+                foundQueryLog = true;
+                assertEquals(rs.getString(BIND_PARAMETERS), loglevel == LogLevel.TRACE ? "value5" : null);
+                assertEquals(rs.getString(USER), System.getProperty("user.name"));
+                assertEquals(rs.getString(CLIENT_IP), InetAddress.getLocalHost().getHostAddress());
+                assertEquals(rs.getString(EXPLAIN_PLAN), QueryUtil.getExplainPlan(explainRS));
+                assertEquals(rs.getString(GLOBAL_SCAN_DETAILS), context.getScan().toJSON());
+                assertEquals(rs.getLong(NO_OF_RESULTS_ITERATED), 1);
+                assertEquals(rs.getString(QUERY), query);
+                assertEquals(rs.getString(QUERY_STATUS), QueryLogState.COMPLETED.toString());
+                assertTrue(System.currentTimeMillis() - rs.getTimestamp(START_TIME).getTime() > delay);
+                assertEquals(rs.getString(TENANT_ID), null);
+                assertTrue(rs.getString(TOTAL_EXECUTION_TIME) != null);
+            }
+        }
+        assertTrue(foundQueryLog);
+        conn.close();
+    }
+    
+    
+    
+    @Test
+    public void testFailedQuery() throws Exception {
+        String tableName = generateUniqueName();
+        Properties props = new Properties();
+        props.setProperty(QueryServices.LOG_LEVEL, LogLevel.DEBUG.name());
+        Connection conn = DriverManager.getConnection(getUrl(), props);
+        assertEquals(conn.unwrap(PhoenixConnection.class).getLogLevel(), LogLevel.DEBUG);
+        // Table does not exists
+        String query = "SELECT * FROM " + tableName;
+
+        try {
+            conn.createStatement().executeQuery(query);
+            fail();
+        } catch (SQLException e) {
+            assertEquals(e.getErrorCode(), SQLExceptionCode.TABLE_UNDEFINED.getErrorCode());
+        }
+        String logQuery = "SELECT * FROM " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_LOG_TABLE + "\"";
+        ResultSet rs = conn.createStatement().executeQuery(logQuery);
+        boolean foundQueryLog = false;
+        int delay = 5000;
+
+        // sleep for sometime to let query log committed
+        Thread.sleep(delay);
+        while (rs.next()) {
+            if (QueryLogState.FAILED.name().equals(rs.getString(QUERY_STATUS))) {
+                foundQueryLog = true;
+                assertEquals(rs.getString(USER), System.getProperty("user.name"));
+                assertEquals(rs.getString(CLIENT_IP), InetAddress.getLocalHost().getHostAddress());
+                assertEquals(rs.getString(EXPLAIN_PLAN), null);
+                assertEquals(rs.getString(GLOBAL_SCAN_DETAILS), null);
+                assertEquals(rs.getLong(NO_OF_RESULTS_ITERATED), 0);
+                assertEquals(rs.getString(QUERY), query);
+                assertTrue(rs.getString(EXCEPTION_TRACE).contains(SQLExceptionCode.TABLE_UNDEFINED.getMessage()));
+                assertTrue(System.currentTimeMillis() - rs.getTimestamp(START_TIME).getTime() > delay);
+                assertTrue(rs.getString(TOTAL_EXECUTION_TIME) != null);
+            }
+        }
+        assertTrue(foundQueryLog);
+        conn.close();
+    }
+
+    private static void createTableAndInsertValues(String tableName, boolean resetGlobalMetricsAfterTableCreate)
+            throws Exception {
+        String ddl = "CREATE TABLE " + tableName + " (K VARCHAR NOT NULL PRIMARY KEY, V VARCHAR)";
+        Connection conn = DriverManager.getConnection(getUrl());
+        conn.createStatement().execute(ddl);
+        // executing 10 upserts/mutations.
+        String dml = "UPSERT INTO " + tableName + " VALUES (?, ?)";
+        PreparedStatement stmt = conn.prepareStatement(dml);
+        for (int i = 1; i <= 10; i++) {
+            stmt.setString(1, "key" + i);
+            stmt.setString(2, "value" + i);
+            stmt.executeUpdate();
+        }
+        conn.commit();
+    }
+    
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
index f8dfd65..34a1312 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
@@ -493,6 +493,8 @@ public class TenantSpecificTablesDDLIT extends BaseTenantSpecificTablesIT {
             assertTrue(rs.next());
             assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, SYSTEM_FUNCTION_TABLE, SYSTEM);
             assertTrue(rs.next());
+            assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE, PTableType.SYSTEM);
+            assertTrue(rs.next());
             assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.TYPE_SEQUENCE, PTableType.SYSTEM);
             assertTrue(rs.next());
             assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE, PTableType.SYSTEM);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
index 39d8525..c105046 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.log.QueryLogger;
 import org.apache.phoenix.monitoring.OverAllQueryMetrics;
 import org.apache.phoenix.monitoring.ReadMetricQueue;
 import org.apache.phoenix.parse.SelectStatement;
@@ -83,6 +84,7 @@ public class StatementContext {
     private Map<SelectStatement, Object> subqueryResults;
     private final ReadMetricQueue readMetricsQueue;
     private final OverAllQueryMetrics overAllQueryMetrics;
+    private QueryLogger queryLogger;
     
     public StatementContext(PhoenixStatement statement) {
         this(statement, new Scan());
@@ -306,5 +308,13 @@ public class StatementContext {
     public OverAllQueryMetrics getOverallQueryMetrics() {
         return overAllQueryMetrics;
     }
+
+    public void setQueryLogger(QueryLogger queryLogger) {
+       this.queryLogger=queryLogger;
+    }
+
+    public QueryLogger getQueryLogger() {
+        return queryLogger;
+    }
     
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index efad1e7..4c4c96f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -76,6 +76,8 @@ public abstract class MetaDataProtocol extends MetaDataService {
     public static final boolean DEFAULT_META_DATA_KEEP_DELETED_CELLS = true;
     public static final int DEFAULT_MAX_STAT_DATA_VERSIONS = 1;
     public static final boolean DEFAULT_STATS_KEEP_DELETED_CELLS = false;
+    public static final int DEFAULT_LOG_VERSIONS = 10;
+    public static final int DEFAULT_LOG_TTL = 7 * 24 * 60 * 60; // 7 days 
     
     // Min system table timestamps for every release.
     public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0 = MIN_TABLE_TIMESTAMP + 3;
@@ -118,7 +120,8 @@ public abstract class MetaDataProtocol extends MetaDataService {
         TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0, "4.14.x");
     }
     
-    public static final String CURRENT_CLIENT_VERSION = PHOENIX_MAJOR_VERSION + "." + PHOENIX_MINOR_VERSION + "." + PHOENIX_PATCH_NUMBER; 
+    public static final String CURRENT_CLIENT_VERSION = PHOENIX_MAJOR_VERSION + "." + PHOENIX_MINOR_VERSION + "." + PHOENIX_PATCH_NUMBER;
+     
     
     // TODO: pare this down to minimum, as we don't need duplicates for both table and column errors, nor should we need
     // a different code for every type of error.

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/iterate/ScanningResultIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ScanningResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ScanningResultIterator.java
index 011feaa..9a31238 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ScanningResultIterator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ScanningResultIterator.java
@@ -17,18 +17,18 @@
  */
 package org.apache.phoenix.iterate;
 
-import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_SCAN_BYTES;
-import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_RPC_CALLS;
-import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_REMOTE_RPC_CALLS;
+import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_BYTES_IN_REMOTE_RESULTS;
+import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_BYTES_REGION_SERVER_RESULTS;
 import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_MILLS_BETWEEN_NEXTS;
 import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_NOT_SERVING_REGION_EXCEPTION;
-import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_BYTES_REGION_SERVER_RESULTS;
-import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_BYTES_IN_REMOTE_RESULTS;
-import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_SCANNED_REGIONS;
-import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_RPC_RETRIES;
+import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_REMOTE_RPC_CALLS;
 import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_REMOTE_RPC_RETRIES;
-import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_ROWS_SCANNED;
 import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_ROWS_FILTERED;
+import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_ROWS_SCANNED;
+import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_RPC_CALLS;
+import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_RPC_RETRIES;
+import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HBASE_COUNT_SCANNED_REGIONS;
+import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_SCAN_BYTES;
 
 import java.io.IOException;
 import java.sql.SQLException;
@@ -93,7 +93,7 @@ public class ScanningResultIterator implements ResultIterator {
             if(scanMetricsMap == null) {
                 return;
             }
-
+            scanMetricsHolder.setScanMetricMap(scanMetricsMap);
             changeMetric(scanMetricsHolder.getCountOfRPCcalls(),
                     scanMetricsMap.get(RPC_CALLS_METRIC_NAME));
             changeMetric(scanMetricsHolder.getCountOfRemoteRPCcalls(),

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
index 2b428c9..d3626f8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
@@ -75,11 +75,11 @@ import org.apache.phoenix.iterate.ParallelIteratorFactory;
 import org.apache.phoenix.iterate.TableResultIterator;
 import org.apache.phoenix.iterate.TableResultIteratorFactory;
 import org.apache.phoenix.jdbc.PhoenixStatement.PhoenixStatementParser;
+import org.apache.phoenix.log.LogLevel;
 import org.apache.phoenix.monitoring.MetricType;
 import org.apache.phoenix.parse.PFunction;
 import org.apache.phoenix.parse.PSchema;
 import org.apache.phoenix.query.ConnectionQueryServices;
-import org.apache.phoenix.query.ConnectionQueryServices.Feature;
 import org.apache.phoenix.query.DelegateConnectionQueryServices;
 import org.apache.phoenix.query.MetaDataMutated;
 import org.apache.phoenix.query.PropertyPolicyProvider;
@@ -168,6 +168,8 @@ public class PhoenixConnection implements Connection, MetaDataMutated, SQLClosea
     private final LinkedBlockingQueue<WeakReference<TableResultIterator>> scannerQueue;
     private TableResultIteratorFactory tableResultIteratorFactory;
     private boolean isRunningUpgrade;
+    private LogLevel logLevel;
+    private Double logSamplingRate;
 
     static {
         Tracing.addTraceMetricsSource();
@@ -378,6 +380,10 @@ public class PhoenixConnection implements Connection, MetaDataMutated, SQLClosea
         this.scannerQueue = new LinkedBlockingQueue<>();
         this.tableResultIteratorFactory = new DefaultTableResultIteratorFactory();
         this.isRunningUpgrade = isRunningUpgrade;
+        this.logLevel= LogLevel.valueOf(this.services.getProps().get(QueryServices.LOG_LEVEL,
+                QueryServicesOptions.DEFAULT_LOGGING_LEVEL));
+        this.logSamplingRate = Double.parseDouble(this.services.getProps().get(QueryServices.LOG_SAMPLE_RATE,
+                QueryServicesOptions.DEFAULT_LOG_SAMPLE_RATE));
         GLOBAL_OPEN_PHOENIX_CONNECTIONS.increment();
     }
 
@@ -648,6 +654,7 @@ public class PhoenixConnection implements Connection, MetaDataMutated, SQLClosea
             } finally {
                 services.removeConnection(this);
             }
+            
         } finally {
             isClosed = true;
             GLOBAL_OPEN_PHOENIX_CONNECTIONS.decrement();
@@ -1274,4 +1281,12 @@ public class PhoenixConnection implements Connection, MetaDataMutated, SQLClosea
         this.isRunningUpgrade = isRunningUpgrade;
     }
 
+    public LogLevel getLogLevel(){
+        return this.logLevel;
+    }
+    
+    public Double getLogSamplingRate(){
+        return this.logSamplingRate;
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index b88b381..9caf7fb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -343,6 +343,24 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
     public static final String USE_STATS_FOR_PARALLELIZATION = "USE_STATS_FOR_PARALLELIZATION";
     public static final byte[] USE_STATS_FOR_PARALLELIZATION_BYTES = Bytes.toBytes(USE_STATS_FOR_PARALLELIZATION);
 
+    
+    //SYSTEM:LOG
+    public static final String SYSTEM_LOG_TABLE = "LOG";
+    public static final String QUERY_ID = "QUERY_ID";
+    public static final String USER = "USER";
+    public static final String CLIENT_IP = "CLIENT_IP";
+    public static final String QUERY = "QUERY";
+    public static final String EXPLAIN_PLAN = "EXPLAIN_PLAN";
+    public static final String TOTAL_EXECUTION_TIME = "TOTAL_EXECUTION_TIME";
+    public static final String NO_OF_RESULTS_ITERATED = "NO_OF_RESULTS_ITERATED";
+    public static final String QUERY_STATUS = "QUERY_STATUS";
+    public static final String EXCEPTION_TRACE = "EXCEPTION_TRACE";
+    public static final String GLOBAL_SCAN_DETAILS = "GLOBAL_SCAN_DETAILS";
+    public static final String SCAN_METRICS_JSON = "SCAN_METRICS_JSON";
+    public static final String START_TIME = "START_TIME";
+    public static final String BIND_PARAMETERS = "BIND_PARAMETERS";
+            
+    
     PhoenixDatabaseMetaData(PhoenixConnection connection) throws SQLException {
         this.emptyResultSet = new PhoenixResultSet(ResultIterator.EMPTY_ITERATOR, RowProjector.EMPTY_PROJECTOR, new StatementContext(new PhoenixStatement(connection), false));
         this.connection = connection;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixPreparedStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixPreparedStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixPreparedStatement.java
index 71ecb8d..914ea33 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixPreparedStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixPreparedStatement.java
@@ -169,7 +169,13 @@ public class PhoenixPreparedStatement extends PhoenixStatement implements Prepar
             throw new SQLExceptionInfo.Builder(SQLExceptionCode.EXECUTE_UPDATE_WITH_NON_EMPTY_BATCH)
             .build().buildException();
         }
-        return execute(statement);
+        if (statement.getOperation().isMutation()) {
+            executeMutation(statement);
+            return false;
+        }
+        executeQuery(statement, createQueryLogger(statement,query));
+        return true;
+        
     }
 
     @Override
@@ -183,7 +189,8 @@ public class PhoenixPreparedStatement extends PhoenixStatement implements Prepar
         if (statement.getOperation().isMutation()) {
             throw new ExecuteQueryNotApplicableException(statement.getOperation());
         }
-        return executeQuery(statement);
+        
+        return executeQuery(statement,createQueryLogger(statement,query));
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java
index d3ec151..153fa08 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java
@@ -50,6 +50,9 @@ import org.apache.phoenix.compile.StatementContext;
 import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.exception.SQLExceptionInfo;
 import org.apache.phoenix.iterate.ResultIterator;
+import org.apache.phoenix.log.QueryLogInfo;
+import org.apache.phoenix.log.QueryLogState;
+import org.apache.phoenix.log.QueryLogger;
 import org.apache.phoenix.monitoring.MetricType;
 import org.apache.phoenix.monitoring.OverAllQueryMetrics;
 import org.apache.phoenix.monitoring.ReadMetricQueue;
@@ -72,6 +75,9 @@ import org.apache.phoenix.schema.types.PVarchar;
 import org.apache.phoenix.util.SQLCloseable;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMap.Builder;
 
 
 
@@ -122,6 +128,14 @@ public class PhoenixResultSet implements ResultSet, SQLCloseable {
     private boolean isClosed = false;
     private boolean wasNull = false;
     private boolean firstRecordRead = false;
+
+    private QueryLogger queryLogger;
+
+    private Long count = 0L;
+
+    private QueryLogState logStatus = QueryLogState.COMPLETED;
+
+    private RuntimeException exception;
     
     public PhoenixResultSet(ResultIterator resultIterator, RowProjector rowProjector, StatementContext ctx) throws SQLException {
         this.rowProjector = rowProjector;
@@ -130,6 +144,7 @@ public class PhoenixResultSet implements ResultSet, SQLCloseable {
         this.statement = context.getStatement();
         this.readMetricsQueue = context.getReadMetricsQueue();
         this.overAllQueryMetrics = context.getOverallQueryMetrics();
+        this.queryLogger = context.getQueryLogger();
     }
     
     @Override
@@ -779,17 +794,39 @@ public class PhoenixResultSet implements ResultSet, SQLCloseable {
             currentRow = scanner.next();
             if (currentRow == null) {
                 close();
+            }else{
+                count++;
             }
             rowProjector.reset();
         } catch (RuntimeException e) {
+            this.logStatus=QueryLogState.FAILED;
             // FIXME: Expression.evaluate does not throw SQLException
             // so this will unwrap throws from that.
+            this.exception = e;
             if (e.getCause() instanceof SQLException) {
                 throw (SQLException) e.getCause();
             }
             throw e;
+        }finally{
+            if (currentRow == null && queryLogger != null ) {
+                if (queryLogger.isDebugEnabled()) {
+                    Builder<QueryLogInfo, Object> queryLogBuilder = ImmutableMap.builder();
+                    queryLogBuilder.put(QueryLogInfo.NO_OF_RESULTS_ITERATED_I, count);
+                    queryLogBuilder.put(QueryLogInfo.TOTAL_EXECUTION_TIME_I,
+                            System.currentTimeMillis() - queryLogger.getStartTime());
+                    queryLogBuilder.put(QueryLogInfo.SCAN_METRICS_JSON_I,
+                            readMetricsQueue.getScanMetricsHolderList().toString());
+                    if (this.exception != null) {
+                        queryLogBuilder.put(QueryLogInfo.EXCEPTION_TRACE_I,
+                                Throwables.getStackTraceAsString(this.exception));
+                    }
+                    readMetricsQueue.getScanMetricsHolderList().clear();
+                    queryLogger.log(logStatus, queryLogBuilder.build());
+                }
+            }
         }
         if (currentRow == null) {
+            
             overAllQueryMetrics.endQuery();
             overAllQueryMetrics.stopResultSetWatch();
         }
@@ -1301,6 +1338,7 @@ public class PhoenixResultSet implements ResultSet, SQLCloseable {
     
     public void resetMetrics() {
         readMetricsQueue.clearMetrics();
+        readMetricsQueue.getScanMetricsHolderList().clear();
         overAllQueryMetrics.reset();
     }
     

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index 4a692d3..f526419 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -92,6 +92,10 @@ import org.apache.phoenix.expression.RowKeyColumnExpression;
 import org.apache.phoenix.iterate.MaterializedResultIterator;
 import org.apache.phoenix.iterate.ParallelScanGrouper;
 import org.apache.phoenix.iterate.ResultIterator;
+import org.apache.phoenix.log.QueryLogInfo;
+import org.apache.phoenix.log.QueryLogState;
+import org.apache.phoenix.log.QueryLogger;
+import org.apache.phoenix.log.QueryLoggerUtil;
 import org.apache.phoenix.optimize.Cost;
 import org.apache.phoenix.parse.AddColumnStatement;
 import org.apache.phoenix.parse.AddJarsStatement;
@@ -186,6 +190,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMap.Builder;
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.math.IntMath;
@@ -269,26 +275,19 @@ public class PhoenixStatement implements Statement, SQLCloseable {
         return new PhoenixResultSet(iterator, projector, context);
     }
     
-    protected boolean execute(final CompilableStatement stmt) throws SQLException {
-        if (stmt.getOperation().isMutation()) {
-            executeMutation(stmt);
-            return false;
-        }
-        executeQuery(stmt);
-        return true;
-    }
-    
     protected QueryPlan optimizeQuery(CompilableStatement stmt) throws SQLException {
         QueryPlan plan = stmt.compilePlan(this, Sequence.ValueOp.VALIDATE_SEQUENCE);
         return connection.getQueryServices().getOptimizer().optimize(this, plan);
     }
     
-    protected PhoenixResultSet executeQuery(final CompilableStatement stmt) throws SQLException {
-      return executeQuery(stmt,true);
+    protected PhoenixResultSet executeQuery(final CompilableStatement stmt, final QueryLogger queryLogger)
+            throws SQLException {
+        return executeQuery(stmt, true, queryLogger);
     }
     private PhoenixResultSet executeQuery(final CompilableStatement stmt,
-        final boolean doRetryOnMetaNotFoundError) throws SQLException {
+        final boolean doRetryOnMetaNotFoundError, final QueryLogger queryLogger) throws SQLException {
         GLOBAL_SELECT_SQL_COUNTER.increment();
+        
         try {
             return CallRunner.run(
                 new CallRunner.CallableThrowable<PhoenixResultSet, SQLException>() {
@@ -297,6 +296,7 @@ public class PhoenixStatement implements Statement, SQLCloseable {
                     final long startTime = System.currentTimeMillis();
                     try {
                         PhoenixConnection conn = getConnection();
+                        
                         if (conn.getQueryServices().isUpgradeRequired() && !conn.isRunningUpgrade()
                                 && stmt.getOperation() != Operation.UPGRADE) {
                             throw new UpgradeRequiredException();
@@ -317,6 +317,13 @@ public class PhoenixStatement implements Statement, SQLCloseable {
                             logger.debug(LogUtil.addCustomAnnotations("Explain plan: " + explainPlan, connection));
                         }
                         StatementContext context = plan.getContext();
+                        context.setQueryLogger(queryLogger);
+                        if(queryLogger.isDebugEnabled()){
+                            Builder<QueryLogInfo, Object> queryLogBuilder = ImmutableMap.builder();
+                            queryLogBuilder.put(QueryLogInfo.EXPLAIN_PLAN_I, QueryUtil.getExplainPlan(resultIterator));
+                            queryLogBuilder.put(QueryLogInfo.GLOBAL_SCAN_DETAILS_I, context.getScan()!=null?context.getScan().toString():null);
+                            queryLogger.log(QueryLogState.COMPILED, queryLogBuilder.build());
+                        }
                         context.getOverallQueryMetrics().startQuery();
                         PhoenixResultSet rs = newResultSet(resultIterator, plan.getProjector(), plan.getContext());
                         resultSets.add(rs);
@@ -338,7 +345,8 @@ public class PhoenixStatement implements Statement, SQLCloseable {
                                 logger.debug("Reloading table "+ e.getTableName()+" data from server");
                             if(new MetaDataClient(connection).updateCache(connection.getTenantId(),
                                 e.getSchemaName(), e.getTableName(), true).wasUpdated()){
-                                return executeQuery(stmt, false);
+                                //TODO we can log retry count and error for debugging in LOG table
+                                return executeQuery(stmt, false, queryLogger);
                             }
                         }
                         throw e;
@@ -358,6 +366,13 @@ public class PhoenixStatement implements Statement, SQLCloseable {
                 }
                 }, PhoenixContextExecutor.inContext());
         }catch (Exception e) {
+            if (queryLogger.isDebugEnabled()) {
+                Builder<QueryLogInfo, Object> queryLogBuilder = ImmutableMap.builder();
+                queryLogBuilder.put(QueryLogInfo.TOTAL_EXECUTION_TIME_I,
+                        System.currentTimeMillis() - queryLogger.getStartTime());
+                queryLogBuilder.put(QueryLogInfo.EXCEPTION_TRACE_I, Throwables.getStackTraceAsString(e));
+                queryLogger.log(QueryLogState.FAILED, queryLogBuilder.build());
+            }
             Throwables.propagateIfInstanceOf(e, SQLException.class);
             Throwables.propagate(e);
             throw new IllegalStateException(); // Can't happen as Throwables.propagate() always throws
@@ -1750,16 +1765,37 @@ public class PhoenixStatement implements Statement, SQLCloseable {
         return compileMutation(stmt, sql);
     }
 
+    public QueryLogger createQueryLogger(CompilableStatement stmt, String sql) throws SQLException {
+        boolean isSystemTable=false;
+        if(stmt instanceof ExecutableSelectStatement){
+            TableNode from = ((ExecutableSelectStatement)stmt).getFrom();
+            if(from instanceof NamedTableNode){
+                String schemaName = ((NamedTableNode)from).getName().getSchemaName();
+                if(schemaName==null){
+                    schemaName=connection.getSchema();
+                }
+                if(PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA.equals(schemaName)){
+                    isSystemTable=true;
+                }
+            }
+        }
+        QueryLogger queryLogger = QueryLogger.getInstance(connection,isSystemTable);
+        QueryLoggerUtil.logInitialDetails(queryLogger, connection.getTenantId(),
+                connection.getQueryServices(), sql, queryLogger.getStartTime(), getParameters());
+        return queryLogger;
+    }
+    
     @Override
     public ResultSet executeQuery(String sql) throws SQLException {
         if (logger.isDebugEnabled()) {
             logger.debug(LogUtil.addCustomAnnotations("Execute query: " + sql, connection));
         }
+        
         CompilableStatement stmt = parseStatement(sql);
         if (stmt.getOperation().isMutation()) {
             throw new ExecuteQueryNotApplicableException(sql);
         }
-        return executeQuery(stmt);
+        return executeQuery(stmt,createQueryLogger(stmt,sql));
     }
 
     @Override
@@ -1795,7 +1831,8 @@ public class PhoenixStatement implements Statement, SQLCloseable {
             flushIfNecessary();
             return false;
         }
-        executeQuery(stmt);
+        
+        executeQuery(stmt,createQueryLogger(stmt,sql));
         return true;
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/log/LogLevel.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/log/LogLevel.java b/phoenix-core/src/main/java/org/apache/phoenix/log/LogLevel.java
new file mode 100644
index 0000000..5792658
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/log/LogLevel.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.log;
+
+public enum LogLevel {
+    OFF, INFO, DEBUG, TRACE
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/log/LogWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/log/LogWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/log/LogWriter.java
new file mode 100644
index 0000000..817f9ec
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/log/LogWriter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.log;
+
+import java.io.IOException;
+import java.sql.SQLException;
+
+/**
+ * Used by the event handler to write RingBufferEvent, this is done in a seperate thread from the application configured
+ * during disruptor
+ */
+public interface LogWriter {
+    /**
+     * Called by ring buffer event handler to write RingBufferEvent
+     * 
+     * @param event
+     * @throws SQLException
+     * @throws IOException
+     */
+    void write(RingBufferEvent event) throws SQLException, IOException;
+
+    /**
+     * will be called when disruptor is getting shutdown
+     * 
+     * @throws IOException
+     */
+
+    void close() throws IOException;
+
+    /**
+     * if writer is closed and cannot write further event
+     * 
+     * @return
+     */
+    boolean isClosed();
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogDetailsEventHandler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogDetailsEventHandler.java b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogDetailsEventHandler.java
new file mode 100644
index 0000000..ee6b2d6
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogDetailsEventHandler.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.log;
+
+import java.sql.SQLException;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.lmax.disruptor.LifecycleAware;
+import com.lmax.disruptor.Sequence;
+import com.lmax.disruptor.SequenceReportingEventHandler;
+
+
+public class QueryLogDetailsEventHandler implements SequenceReportingEventHandler<RingBufferEvent>, LifecycleAware {
+    private Sequence sequenceCallback;
+    private LogWriter logWriter;
+
+    public QueryLogDetailsEventHandler(Configuration configuration) throws SQLException{
+        this.logWriter = new TableLogWriter(configuration);
+    }
+    
+    @Override
+    public void setSequenceCallback(final Sequence sequenceCallback) {
+        this.sequenceCallback = sequenceCallback;
+    }
+
+    @Override
+    public void onEvent(final RingBufferEvent event, final long sequence, final boolean endOfBatch) throws Exception {
+        logWriter.write(event);
+        event.clear();
+    }
+
+    @Override
+    public void onStart() {
+    }
+
+    @Override
+    public void onShutdown() {
+        try {
+            if (logWriter != null) {
+                logWriter.close();
+            }
+        } catch (Exception e) {
+            //Ignore
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogInfo.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogInfo.java b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogInfo.java
new file mode 100644
index 0000000..87de267
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogInfo.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.log;
+
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BIND_PARAMETERS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLIENT_IP;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXCEPTION_TRACE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXPLAIN_PLAN;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GLOBAL_SCAN_DETAILS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NO_OF_RESULTS_ITERATED;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_ID;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_STATUS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCAN_METRICS_JSON;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_TIME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TOTAL_EXECUTION_TIME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USER;
+
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PLong;
+import org.apache.phoenix.schema.types.PTimestamp;
+import org.apache.phoenix.schema.types.PVarchar;
+
+
+public enum QueryLogInfo {
+    
+    CLIENT_IP_I(CLIENT_IP, QueryLogState.STARTED, LogLevel.INFO, PVarchar.INSTANCE),
+    QUERY_I(QUERY,QueryLogState.STARTED, LogLevel.INFO,PVarchar.INSTANCE),
+    BIND_PARAMETERS_I(BIND_PARAMETERS,QueryLogState.STARTED, LogLevel.TRACE,PVarchar.INSTANCE),
+    QUERY_ID_I(QUERY_ID,QueryLogState.STARTED, LogLevel.INFO,PVarchar.INSTANCE),
+    TENANT_ID_I(TENANT_ID,QueryLogState.STARTED, LogLevel.INFO,PVarchar.INSTANCE),
+    START_TIME_I(START_TIME,QueryLogState.STARTED, LogLevel.INFO,PTimestamp.INSTANCE),
+    USER_I(USER,QueryLogState.STARTED, LogLevel.INFO,PVarchar.INSTANCE),
+    EXPLAIN_PLAN_I(EXPLAIN_PLAN,QueryLogState.COMPILED, LogLevel.DEBUG,PVarchar.INSTANCE),
+    GLOBAL_SCAN_DETAILS_I(GLOBAL_SCAN_DETAILS,QueryLogState.COMPILED, LogLevel.DEBUG,PVarchar.INSTANCE),
+    NO_OF_RESULTS_ITERATED_I(NO_OF_RESULTS_ITERATED,QueryLogState.COMPLETED, LogLevel.DEBUG,PLong.INSTANCE),
+    EXCEPTION_TRACE_I(EXCEPTION_TRACE,QueryLogState.COMPLETED, LogLevel.DEBUG,PVarchar.INSTANCE),
+    QUERY_STATUS_I(QUERY_STATUS,QueryLogState.COMPLETED, LogLevel.DEBUG,PVarchar.INSTANCE),
+    TOTAL_EXECUTION_TIME_I(TOTAL_EXECUTION_TIME,QueryLogState.COMPLETED, LogLevel.DEBUG,PLong.INSTANCE),
+    SCAN_METRICS_JSON_I(SCAN_METRICS_JSON,QueryLogState.COMPLETED, LogLevel.DEBUG,PVarchar.INSTANCE);
+    
+    public final String columnName;
+    public final QueryLogState logState;
+    public final LogLevel logLevel;
+    public final PDataType dataType;
+
+    private QueryLogInfo(String columnName, QueryLogState logState, LogLevel logLevel, PDataType dataType) {
+        this.columnName = columnName;
+        this.logState=logState;
+        this.logLevel=logLevel;
+        this.dataType=dataType;
+    }
+
+    public String getColumnName() {
+        return columnName;
+    }
+
+    public QueryLogState getLogState() {
+        return logState;
+    }
+
+    public LogLevel getLogLevel() {
+        return logLevel;
+    }
+
+    public PDataType getDataType() {
+        return dataType;
+    }
+    
+    
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogState.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogState.java b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogState.java
new file mode 100644
index 0000000..e27f0e8
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogState.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.log;
+
+public enum QueryLogState {
+    STARTED, PLAN, COMPILED, EXECUTION, COMPLETED,FAILED 
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogger.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogger.java b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogger.java
new file mode 100644
index 0000000..b2fb235
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLogger.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.log;
+
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+
+import com.google.common.collect.ImmutableMap;
+
+import io.netty.util.internal.ThreadLocalRandom;
+
+/*
+ * Wrapper for query translator
+ */
+public class QueryLogger {
+    private final ThreadLocal<RingBufferEventTranslator> threadLocalTranslator = new ThreadLocal<>();
+    private QueryLoggerDisruptor queryDisruptor;
+    private String queryId;
+    private Long startTime;
+    private LogLevel logLevel;
+    private static final Log LOG = LogFactory.getLog(QueryLoggerDisruptor.class);
+    
+    private QueryLogger(PhoenixConnection connection) {
+        this.queryId = UUID.randomUUID().toString();
+        this.queryDisruptor = connection.getQueryServices().getQueryDisruptor();
+        this.startTime = System.currentTimeMillis();
+        logLevel = connection.getLogLevel();
+    }
+    
+    private QueryLogger() {
+        logLevel = LogLevel.OFF;
+    }
+    
+    private RingBufferEventTranslator getCachedTranslator() {
+        RingBufferEventTranslator result = threadLocalTranslator.get();
+        if (result == null) {
+            result = new RingBufferEventTranslator(queryId);
+            threadLocalTranslator.set(result);
+        }
+        return result;
+    }
+    
+    private static final QueryLogger NO_OP_INSTANCE = new QueryLogger() {
+        @Override
+        public void log(QueryLogState logState, ImmutableMap<QueryLogInfo, Object> map) {
+
+        }
+        
+        @Override
+        public boolean isDebugEnabled(){
+            return false;
+        }
+        
+        @Override
+        public boolean isInfoEnabled(){
+            return false;
+        }
+    };
+
+    public static QueryLogger getInstance(PhoenixConnection connection, boolean isSystemTable) {
+        if (connection.getLogLevel() == LogLevel.OFF || isSystemTable || ThreadLocalRandom.current()
+                .nextDouble() > connection.getLogSamplingRate()) { return NO_OP_INSTANCE; }
+        return new QueryLogger(connection);
+    }
+
+    /**
+     * Add query log in the table, columns will be logged depending upon the connection logLevel
+     * @param logState State of the query
+     * @param map Value of the map should be in format of the corresponding data type 
+     */
+    public void log(QueryLogState logState, ImmutableMap<QueryLogInfo, Object> map) {
+        final RingBufferEventTranslator translator = getCachedTranslator();
+        translator.setQueryInfo(logState, map, logLevel);
+        publishLogs(translator);
+    }
+    
+    private boolean publishLogs(RingBufferEventTranslator translator) {
+        if (queryDisruptor == null) { return false; }
+        boolean isLogged = queryDisruptor.tryPublish(translator);
+        if (!isLogged && LOG.isDebugEnabled()) {
+            LOG.debug("Unable to write query log in table as ring buffer queue is full!!");
+        }
+        return isLogged;
+    }
+
+    /**
+     * Start time when the logger was started, if {@link LogLevel#OFF} then it's the current time
+     */
+    public Long getStartTime() {
+        return startTime != null ? startTime : System.currentTimeMillis();
+    }
+    
+    /**
+     *  Is debug logging currently enabled?
+     *  Call this method to prevent having to perform expensive operations (for example, String concatenation) when the log level is more than debug.
+     */
+    public boolean isDebugEnabled(){
+        return isLevelEnabled(LogLevel.DEBUG);
+    }
+    
+    private boolean isLevelEnabled(LogLevel logLevel){
+        return this.logLevel != null ? logLevel.ordinal() <= this.logLevel.ordinal() : false;
+    }
+    
+    /**
+     * Is Info logging currently enabled?
+     * Call this method to prevent having to perform expensive operations (for example, String concatenation) when the log level is more than info.
+     * @return
+     */
+    public boolean isInfoEnabled(){
+        return isLevelEnabled(LogLevel.INFO);
+    }
+
+    /**
+     * Return queryId of the current query logger , needed by the application 
+     * to correlate with the logging table.
+     * Eg(usage):-
+     * StatementContext context = ((PhoenixResultSet)rs).getContext();
+     * String queryId = context.getQueryLogger().getQueryId();
+     * 
+     * @return
+     */
+    public String getQueryId() {
+        return this.queryId;
+    }
+    
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerDefaultExceptionHandler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerDefaultExceptionHandler.java b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerDefaultExceptionHandler.java
new file mode 100644
index 0000000..e9ae6bd
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerDefaultExceptionHandler.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.log;
+
+import com.lmax.disruptor.ExceptionHandler;
+
+class QueryLoggerDefaultExceptionHandler implements ExceptionHandler<RingBufferEvent> {
+
+    @Override
+    public void handleEventException(Throwable ex, long sequence, RingBufferEvent event) {
+        final StringBuilder sb = new StringBuilder(512);
+        sb.append("Query Logger error handling event seq=").append(sequence).append(", value='");
+        try {
+            sb.append(event);
+        } catch (final Exception ignored) {
+            sb.append("[ERROR calling ").append(event.getClass()).append(".toString(): ");
+            sb.append(ignored).append("]");
+        }
+        sb.append("':");
+        System.err.println(sb);
+        ex.printStackTrace();
+    }
+
+    @Override
+    public void handleOnStartException(final Throwable throwable) {
+        System.err.println("QueryLogger error starting:");
+        throwable.printStackTrace();
+    }
+
+    @Override
+    public void handleOnShutdownException(final Throwable throwable) {
+        System.err.println("QueryLogger error shutting down:");
+        throwable.printStackTrace();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerDisruptor.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerDisruptor.java b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerDisruptor.java
new file mode 100644
index 0000000..b548d6c
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerDisruptor.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.log;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.phoenix.query.QueryServices;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.lmax.disruptor.BlockingWaitStrategy;
+import com.lmax.disruptor.EventTranslator;
+import com.lmax.disruptor.ExceptionHandler;
+import com.lmax.disruptor.TimeoutException;
+import com.lmax.disruptor.WaitStrategy;
+import com.lmax.disruptor.dsl.Disruptor;
+import com.lmax.disruptor.dsl.ProducerType;
+
+public class QueryLoggerDisruptor implements Closeable{
+    
+    private volatile Disruptor<RingBufferEvent> disruptor;
+    private boolean isClosed = false;
+    //number of elements to create within the ring buffer.
+    private static final int RING_BUFFER_SIZE = 256 * 1024;
+    private static final Log LOG = LogFactory.getLog(QueryLoggerDisruptor.class);
+    private static final String DEFAULT_WAIT_STRATEGY = BlockingWaitStrategy.class.getName();
+    
+    public QueryLoggerDisruptor(Configuration configuration) throws SQLException{
+        WaitStrategy waitStrategy;
+        try {
+            waitStrategy = (WaitStrategy)Class
+                    .forName(configuration.get(QueryServices.LOG_BUFFER_WAIT_STRATEGY, DEFAULT_WAIT_STRATEGY)).newInstance();
+        } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
+            throw new SQLException(e); 
+        }
+        
+        ThreadFactory threadFactory = new ThreadFactoryBuilder()
+                .setNameFormat("QueryLogger" + "-thread-%s")
+                .setDaemon(true)
+                .setThreadFactory(new ThreadFactory() {
+                    @Override
+                    public Thread newThread(Runnable r) {
+                        final Thread result = Executors.defaultThreadFactory().newThread(r);
+                        result.setContextClassLoader(QueryLoggerDisruptor.class.getClass().getClassLoader());
+                        return result;
+                    }
+                })
+                .build();
+        disruptor = new Disruptor<RingBufferEvent>(RingBufferEvent.FACTORY,
+                configuration.getInt(QueryServices.LOG_BUFFER_SIZE, RING_BUFFER_SIZE), threadFactory, ProducerType.MULTI,
+                waitStrategy);
+        final ExceptionHandler<RingBufferEvent> errorHandler = new QueryLoggerDefaultExceptionHandler();
+        disruptor.setDefaultExceptionHandler(errorHandler);
+
+        final QueryLogDetailsEventHandler[] handlers = { new QueryLogDetailsEventHandler(configuration) };
+        disruptor.handleEventsWith(handlers);
+        LOG.info("Starting  QueryLoggerDisruptor for with ringbufferSize=" + disruptor.getRingBuffer().getBufferSize()
+                + ", waitStrategy=" + waitStrategy.getClass().getSimpleName() + ", " + "exceptionHandler="
+                + errorHandler + "...");
+        disruptor.start();
+        
+    }
+    
+    /**
+     * Attempts to publish an event by translating (write) data representations into events claimed from the RingBuffer.
+     * @param translator
+     * @return
+     */
+    public boolean tryPublish(final EventTranslator<RingBufferEvent> translator) {
+        if(isClosed()){
+            return false;
+        }
+        return disruptor.getRingBuffer().tryPublishEvent(translator);
+    }
+    
+
+    public boolean isClosed() {
+        return isClosed ;
+    }
+
+    @Override
+    public void close() throws IOException {
+        isClosed = true;
+        LOG.info("Shutting down QueryLoggerDisruptor..");
+        try {
+            //we can wait for 2 seconds, so that backlog can be committed
+            disruptor.shutdown(2, TimeUnit.SECONDS);
+        } catch (TimeoutException e) {
+            throw new IOException(e);
+        }
+
+    }
+    
+    
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerUtil.java
new file mode 100644
index 0000000..2f22931
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerUtil.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.log;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.phoenix.query.ConnectionQueryServices;
+import org.apache.phoenix.schema.PName;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMap.Builder;
+
+public class QueryLoggerUtil {
+
+    public static void logInitialDetails(QueryLogger queryLogger, PName tenantId,
+            ConnectionQueryServices queryServices, String query, long startTime, List<Object> bindParameters) {
+        queryLogger.log(QueryLogState.STARTED,
+                getInitialDetails(tenantId, queryServices, query, startTime, bindParameters));
+
+    }
+
+    private static ImmutableMap<QueryLogInfo, Object> getInitialDetails(PName tenantId,
+            ConnectionQueryServices queryServices, String query, long startTime, List<Object> bindParameters) {
+        Builder<QueryLogInfo, Object> queryLogBuilder = ImmutableMap.builder();
+        String clientIP;
+        try {
+            clientIP = InetAddress.getLocalHost().getHostAddress();
+        } catch (UnknownHostException e) {
+            clientIP = "UnknownHost";
+        }
+        queryLogBuilder.put(QueryLogInfo.CLIENT_IP_I, clientIP);
+        queryLogBuilder.put(QueryLogInfo.QUERY_I, query);
+        queryLogBuilder.put(QueryLogInfo.START_TIME_I, startTime);
+        if (bindParameters != null) {
+            queryLogBuilder.put(QueryLogInfo.BIND_PARAMETERS_I, StringUtils.join(bindParameters,","));
+        }
+        if (tenantId != null) {
+            queryLogBuilder.put(QueryLogInfo.TENANT_ID_I, tenantId.getString());
+        }
+        queryLogBuilder.put(QueryLogInfo.USER_I, queryServices.getUserName() != null ? queryServices.getUserName()
+                : queryServices.getUser().getShortName());
+        return queryLogBuilder.build();
+    }
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/log/RingBufferEvent.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/log/RingBufferEvent.java b/phoenix-core/src/main/java/org/apache/phoenix/log/RingBufferEvent.java
new file mode 100644
index 0000000..96e4bf9
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/log/RingBufferEvent.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.log;
+
+import com.google.common.collect.ImmutableMap;
+import com.lmax.disruptor.EventFactory;
+
+ class RingBufferEvent {
+    private String queryId;
+    private QueryLogState logState;
+    private LogLevel connectionLogLevel;
+    private ImmutableMap<QueryLogInfo, Object> queryInfo;
+    
+    public static final Factory FACTORY = new Factory();
+    
+    /**
+     * Creates the events that will be put in the RingBuffer.
+     */
+    private static class Factory implements EventFactory<RingBufferEvent> {
+        @Override
+        public RingBufferEvent newInstance() {
+            final RingBufferEvent result = new RingBufferEvent();
+            return result;
+        }
+    }
+
+    public void clear() {
+        this.logState=null;
+        this.queryInfo=null;
+        this.queryId=null;
+    }
+
+   
+    public String getQueryId() {
+        return queryId;
+    }
+
+    public static Factory getFactory() {
+        return FACTORY;
+    }
+    
+    public QueryLogState getLogState() {
+        return logState;
+    }
+
+    public void setQueryInfo(ImmutableMap<QueryLogInfo, Object> queryInfo) {
+        this.queryInfo=queryInfo;
+        
+    }
+
+    public void setQueryId(String queryId) {
+        this.queryId=queryId;
+        
+    }
+
+    public ImmutableMap<QueryLogInfo, Object> getQueryInfo() {
+        return queryInfo;
+        
+    }
+
+    public void setLogState(QueryLogState logState) {
+        this.logState=logState;
+        
+    }
+
+
+    public LogLevel getConnectionLogLevel() {
+        return connectionLogLevel;
+    }
+
+
+    public void setConnectionLogLevel(LogLevel connectionLogLevel) {
+        this.connectionLogLevel = connectionLogLevel;
+    }
+
+    
+
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bc03684/phoenix-core/src/main/java/org/apache/phoenix/log/RingBufferEventTranslator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/log/RingBufferEventTranslator.java b/phoenix-core/src/main/java/org/apache/phoenix/log/RingBufferEventTranslator.java
new file mode 100644
index 0000000..653ddd6
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/log/RingBufferEventTranslator.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.log;
+
+import com.google.common.collect.ImmutableMap;
+import com.lmax.disruptor.EventTranslator;
+
+class RingBufferEventTranslator implements EventTranslator<RingBufferEvent> {
+    private String queryId;
+    private QueryLogState logState;
+    private ImmutableMap<QueryLogInfo, Object> queryInfo;
+    private LogLevel connectionLogLevel;
+    
+    public RingBufferEventTranslator(String queryId) {
+        this.queryId=queryId;
+    }
+
+    @Override
+    public void translateTo(RingBufferEvent event, long sequence) {
+        event.setQueryId(queryId);
+        event.setQueryInfo(queryInfo);
+        event.setLogState(logState);
+        event.setConnectionLogLevel(connectionLogLevel);
+        clear();
+    }
+
+    private void clear() {
+        setQueryInfo(null,null,null);
+    }
+   
+    public void setQueryInfo(QueryLogState logState, ImmutableMap<QueryLogInfo, Object> queryInfo,
+            LogLevel connectionLogLevel) {
+        this.queryInfo = queryInfo;
+        this.logState = logState;
+        this.connectionLogLevel = connectionLogLevel;
+    }
+
+}


[20/50] [abbrv] phoenix git commit: PHOENIX-4668 Remove unnecessary table descriptor modification for SPLIT_POLICY column (Chinmay Kulkarni)

Posted by td...@apache.org.
PHOENIX-4668 Remove unnecessary table descriptor modification for SPLIT_POLICY column (Chinmay Kulkarni)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a5770a5a
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a5770a5a
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a5770a5a

Branch: refs/heads/system-catalog
Commit: a5770a5a4d41bf786d3d8dd7ca9088a00418b666
Parents: 87fdda8
Author: James Taylor <jt...@salesforce.com>
Authored: Fri Apr 13 14:19:15 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Apr 13 14:19:15 2018 -0700

----------------------------------------------------------------------
 .../phoenix/query/ConnectionQueryServicesImpl.java       | 11 -----------
 1 file changed, 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5770a5a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 8b328d3..4c1902d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -1130,10 +1130,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                         PBoolean.INSTANCE.toObject(newDesc.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_BYTES)))) {
                     newDesc.setValue(HTableDescriptor.SPLIT_POLICY, IndexRegionSplitPolicy.class.getName());
                 }
-                // Remove the splitPolicy attribute to prevent HBASE-12570
-                if (isMetaTable) {
-                    newDesc.remove(HTableDescriptor.SPLIT_POLICY);
-                }
                 try {
                     if (splits == null) {
                         admin.createTable(newDesc);
@@ -1150,13 +1146,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                 }
                 if (isMetaTable && !isUpgradeRequired()) {
                     checkClientServerCompatibility(SchemaUtil.getPhysicalName(SYSTEM_CATALOG_NAME_BYTES, this.getProps()).getName());
-                    /*
-                     * Now we modify the table to add the split policy, since we know that the client and
-                     * server and compatible. This works around HBASE-12570 which causes the cluster to be
-                     * brought down.
-                     */
-                    newDesc.setValue(HTableDescriptor.SPLIT_POLICY, MetaDataSplitPolicy.class.getName());
-                    modifyTable(physicalTableName, newDesc, true);
                 }
                 return null;
             } else {


[46/50] [abbrv] phoenix git commit: PHOENIX-4711 Unable to set property on table with VARBINARY as last column

Posted by td...@apache.org.
PHOENIX-4711 Unable to set property on table with VARBINARY as last column


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a18fd1e1
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a18fd1e1
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a18fd1e1

Branch: refs/heads/system-catalog
Commit: a18fd1e133f5a2785e0ade236d72028b3d6854da
Parents: 49c0232
Author: James Taylor <jt...@salesforce.com>
Authored: Thu Apr 26 17:13:48 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Apr 27 11:41:56 2018 -0700

----------------------------------------------------------------------
 .../apache/phoenix/end2end/AlterTableIT.java    | 12 ++++++-----
 .../apache/phoenix/schema/MetaDataClient.java   | 22 +++++++++++---------
 2 files changed, 19 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/a18fd1e1/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
index dd895dc..b1949ed 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
@@ -116,12 +116,14 @@ public class AlterTableIT extends ParallelStatsDisabledIT {
         Connection conn = DriverManager.getConnection(getUrl(), props);
         conn.setAutoCommit(false);
 
-        try {
-            String ddl = "CREATE TABLE  " + dataTableFullName +
-                    "  (a_string varchar not null, a_binary varbinary not null, col1 integer" +
-                    "  CONSTRAINT pk PRIMARY KEY (a_string, a_binary)) " + tableDDLOptions;
-            createTestTable(getUrl(), ddl);
+        String ddl = "CREATE TABLE  " + dataTableFullName +
+                "  (a_string varchar not null, a_binary varbinary not null, col1 integer" +
+                "  CONSTRAINT pk PRIMARY KEY (a_string, a_binary)) " + tableDDLOptions;
+        createTestTable(getUrl(), ddl);
+        
+        conn.createStatement().execute("ALTER TABLE " + dataTableFullName + " SET DISABLE_WAL = true");
 
+        try {
             ddl = "ALTER TABLE " + dataTableFullName + " ADD b_string VARCHAR NULL PRIMARY KEY";
             PreparedStatement stmt = conn.prepareStatement(ddl);
             stmt.execute();

http://git-wip-us.apache.org/repos/asf/phoenix/blob/a18fd1e1/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index b15072a..a3d2baf 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -3259,16 +3259,18 @@ public class MetaDataClient {
                 int position = table.getColumns().size();
 
                 List<PColumn> currentPKs = table.getPKColumns();
-                PColumn lastPK = currentPKs.get(currentPKs.size()-1);
-                // Disallow adding columns if the last column is VARBIANRY.
-                if (lastPK.getDataType() == PVarbinary.INSTANCE || lastPK.getDataType().isArrayType()) {
-                    throw new SQLExceptionInfo.Builder(SQLExceptionCode.VARBINARY_LAST_PK)
-                    .setColumnName(lastPK.getName().getString()).build().buildException();
-                }
-                // Disallow adding columns if last column is fixed width and nullable.
-                if (lastPK.isNullable() && lastPK.getDataType().isFixedWidth()) {
-                    throw new SQLExceptionInfo.Builder(SQLExceptionCode.NULLABLE_FIXED_WIDTH_LAST_PK)
-                    .setColumnName(lastPK.getName().getString()).build().buildException();
+                if (numCols > 0) {
+                    PColumn lastPK = currentPKs.get(currentPKs.size()-1);
+                    // Disallow adding columns if the last column is VARBIANRY.
+                    if (lastPK.getDataType() == PVarbinary.INSTANCE || lastPK.getDataType().isArrayType()) {
+                        throw new SQLExceptionInfo.Builder(SQLExceptionCode.VARBINARY_LAST_PK)
+                        .setColumnName(lastPK.getName().getString()).build().buildException();
+                    }
+                    // Disallow adding columns if last column is fixed width and nullable.
+                    if (lastPK.isNullable() && lastPK.getDataType().isFixedWidth()) {
+                        throw new SQLExceptionInfo.Builder(SQLExceptionCode.NULLABLE_FIXED_WIDTH_LAST_PK)
+                        .setColumnName(lastPK.getName().getString()).build().buildException();
+                    }
                 }
 
 


[45/50] [abbrv] phoenix git commit: PHOENIX-4710 Don't set KEEP_DELETED_CELLS or VERSIONS for SYSTEM.LOG

Posted by td...@apache.org.
PHOENIX-4710 Don't set KEEP_DELETED_CELLS or VERSIONS for SYSTEM.LOG


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/49c02328
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/49c02328
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/49c02328

Branch: refs/heads/system-catalog
Commit: 49c02328a0627939838b51b4440abe9427244820
Parents: 6c1a624
Author: James Taylor <jt...@salesforce.com>
Authored: Thu Apr 26 12:48:14 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Apr 27 11:41:56 2018 -0700

----------------------------------------------------------------------
 .../org/apache/phoenix/coprocessor/MetaDataProtocol.java    | 1 -
 .../apache/phoenix/query/ConnectionQueryServicesImpl.java   | 9 ++-------
 .../phoenix/query/ConnectionlessQueryServicesImpl.java      | 7 +------
 3 files changed, 3 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/49c02328/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index 36d6f0d..62b701d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -72,7 +72,6 @@ public abstract class MetaDataProtocol extends MetaDataService {
     public static final long MIN_SYSTEM_TABLE_MIGRATION_TIMESTAMP = 0;
     public static final String MIGRATION_IN_PROGRESS = "MigrationInProgress";
 
-    public static final int DEFAULT_LOG_VERSIONS = 10;
     public static final int DEFAULT_LOG_TTL = 7 * 24 * 60 * 60; // 7 days 
     
     // Min system table timestamps for every release.

http://git-wip-us.apache.org/repos/asf/phoenix/blob/49c02328/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 8866ced..d841f0b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -170,8 +170,8 @@ import org.apache.phoenix.exception.RetriableUpgradeException;
 import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.exception.SQLExceptionInfo;
 import org.apache.phoenix.exception.UpgradeInProgressException;
-import org.apache.phoenix.exception.UpgradeRequiredException;
 import org.apache.phoenix.exception.UpgradeNotRequiredException;
+import org.apache.phoenix.exception.UpgradeRequiredException;
 import org.apache.phoenix.execute.MutationState;
 import org.apache.phoenix.hbase.index.IndexRegionSplitPolicy;
 import org.apache.phoenix.hbase.index.Indexer;
@@ -2469,7 +2469,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
 
     // Available for testing
     protected String getLogTableDDL() {
-        return setSystemLogDDLProperties(QueryConstants.CREATE_LOG_METADATA);
+        return QueryConstants.CREATE_LOG_METADATA;
     }
 
     private String setSystemDDLProperties(String ddl) {
@@ -2478,11 +2478,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
           props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS));
     }
 
-    private String setSystemLogDDLProperties(String ddl) {
-        return String.format(ddl,
-          props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS));
-    }
-
     @Override
     public void init(final String url, final Properties props) throws SQLException {
         try {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/49c02328/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index aba9600..3531a87 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -164,7 +164,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
     }
 
     protected String getLogTableDDL() {
-        return setSystemLogDDLProperties(QueryConstants.CREATE_LOG_METADATA);
+        return QueryConstants.CREATE_LOG_METADATA;
     }
 
     private String setSystemDDLProperties(String ddl) {
@@ -173,11 +173,6 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
           props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS));
     }
 
-    private String setSystemLogDDLProperties(String ddl) {
-        return String.format(ddl,
-          props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS));
-    }
-
     @Override
     public ConnectionQueryServices getChildQueryServices(ImmutableBytesWritable childId) {
         return this; // Just reuse the same query services


[24/50] [abbrv] phoenix git commit: PHOENIX-4690 GroupBy expressions should follow the order of PK Columns if GroupBy is orderPreserving

Posted by td...@apache.org.
PHOENIX-4690 GroupBy expressions should follow the order of PK Columns if GroupBy is orderPreserving


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/153b357d
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/153b357d
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/153b357d

Branch: refs/heads/system-catalog
Commit: 153b357d5b9b8663c13e22b42227dffe12d176f3
Parents: 34f93d7
Author: chenglei <ch...@apache.org>
Authored: Wed Apr 18 16:42:31 2018 +0800
Committer: chenglei <ch...@apache.org>
Committed: Wed Apr 18 16:42:31 2018 +0800

----------------------------------------------------------------------
 .../org/apache/phoenix/end2end/AggregateIT.java | 107 +++++++++++++++---
 .../org/apache/phoenix/end2end/OrderByIT.java   |  18 +--
 .../apache/phoenix/end2end/join/SubqueryIT.java |   8 +-
 .../join/SubqueryUsingSortMergeJoinIT.java      |  12 +-
 .../apache/phoenix/compile/GroupByCompiler.java |  19 +++-
 .../phoenix/compile/OrderPreservingTracker.java |  13 +++
 .../phoenix/compile/QueryCompilerTest.java      | 110 +++++++++++++++++++
 .../java/org/apache/phoenix/util/TestUtil.java  |  20 ++++
 8 files changed, 267 insertions(+), 40 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/153b357d/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateIT.java
index b6c5a06..2059311 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateIT.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.apache.phoenix.util.TestUtil.assertResultSet;
 
 import java.io.IOException;
 import java.sql.Connection;
@@ -1016,20 +1017,100 @@ public class AggregateIT extends ParallelStatsDisabledIT {
         }
     }
 
-    private void assertResultSet(ResultSet rs,Object[][] rows) throws Exception {
-        for(int rowIndex=0;rowIndex<rows.length;rowIndex++) {
-            assertTrue(rs.next());
-            for(int columnIndex=1;columnIndex<= rows[rowIndex].length;columnIndex++) {
-                Object realValue=rs.getObject(columnIndex);
-                Object expectedValue=rows[rowIndex][columnIndex-1];
-                if(realValue==null) {
-                    assertTrue(expectedValue==null);
-                }
-                else {
-                    assertTrue(realValue.equals(expectedValue));
-                }
+    @Test
+    public void testGroupByOrderMatchPkColumnOrderBug4690() throws Exception {
+        this.doTestGroupByOrderMatchPkColumnOrderBug4690(false, false);
+        this.doTestGroupByOrderMatchPkColumnOrderBug4690(false, true);
+        this.doTestGroupByOrderMatchPkColumnOrderBug4690(true, false);
+        this.doTestGroupByOrderMatchPkColumnOrderBug4690(true, true);
+    }
+
+    private void doTestGroupByOrderMatchPkColumnOrderBug4690(boolean desc ,boolean salted) throws Exception {
+        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+        Connection conn = null;
+        try {
+            conn = DriverManager.getConnection(getUrl(), props);
+            String tableName = generateUniqueName();
+            String sql = "create table " + tableName + "( "+
+                    " pk1 integer not null , " +
+                    " pk2 integer not null, " +
+                    " pk3 integer not null," +
+                    " pk4 integer not null,"+
+                    " v integer, " +
+                    " CONSTRAINT TEST_PK PRIMARY KEY ( "+
+                       "pk1 "+(desc ? "desc" : "")+", "+
+                       "pk2 "+(desc ? "desc" : "")+", "+
+                       "pk3 "+(desc ? "desc" : "")+", "+
+                       "pk4 "+(desc ? "desc" : "")+
+                    " )) "+(salted ? "SALT_BUCKETS =4" : "split on(2)");
+            conn.createStatement().execute(sql);
+
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,8,10,20,30)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,8,11,21,31)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,5 ,22,32)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,6 ,12,33)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,6 ,13,34)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,7 ,8,35)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,3,15,25,35)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,7,16,26,36)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,7,17,27,37)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,2,18,28,38)");
+            conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,2,19,29,39)");
+            conn.commit();
+
+            sql = "select pk2,pk1,count(v) from " + tableName + " group by pk2,pk1 order by pk2,pk1";
+            ResultSet rs = conn.prepareStatement(sql).executeQuery();
+            assertResultSet(rs, new Object[][]{{2,3,2L},{3,2,1L},{7,2,2L},{8,1,2L},{9,1,4L}});
+
+            sql = "select pk1,pk2,count(v) from " + tableName + " group by pk2,pk1 order by pk1,pk2";
+            rs = conn.prepareStatement(sql).executeQuery();
+            assertResultSet(rs, new Object[][]{{1,8,2L},{1,9,4L},{2,3,1L},{2,7,2L},{3,2,2L}});
+
+            sql = "select pk2,pk1,count(v) from " + tableName + " group by pk2,pk1 order by pk2 desc,pk1 desc";
+            rs = conn.prepareStatement(sql).executeQuery();
+            assertResultSet(rs, new Object[][]{{9,1,4L},{8,1,2L},{7,2,2L},{3,2,1L},{2,3,2L}});
+
+            sql = "select pk1,pk2,count(v) from " + tableName + " group by pk2,pk1 order by pk1 desc,pk2 desc";
+            rs = conn.prepareStatement(sql).executeQuery();
+            assertResultSet(rs, new Object[][]{{3,2,2L},{2,7,2L},{2,3,1L},{1,9,4L},{1,8,2L}});
+
+
+            sql = "select pk3,pk2,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk3,pk2";
+            rs = conn.prepareStatement(sql).executeQuery();
+            assertResultSet(rs, new Object[][]{{5,9,1L},{6,9,2L},{7,9,1L},{10,8,1L},{11,8,1L}});
+
+            sql = "select pk2,pk3,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk2,pk3";
+            rs = conn.prepareStatement(sql).executeQuery();
+            assertResultSet(rs, new Object[][]{{8,10,1L},{8,11,1L},{9,5,1L},{9,6,2L},{9,7,1L}});
+
+            sql = "select pk3,pk2,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk3 desc,pk2 desc";
+            rs = conn.prepareStatement(sql).executeQuery();
+            assertResultSet(rs, new Object[][]{{11,8,1L},{10,8,1L},{7,9,1L},{6,9,2L},{5,9,1L}});
+
+            sql = "select pk2,pk3,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk2 desc,pk3 desc";
+            rs = conn.prepareStatement(sql).executeQuery();
+            assertResultSet(rs, new Object[][]{{9,7,1L},{9,6,2L},{9,5,1L},{8,11,1L},{8,10,1L}});
+
+
+            sql = "select pk4,pk3,pk1,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk4,pk3,pk1";
+            rs = conn.prepareStatement(sql).executeQuery();
+            assertResultSet(rs, new Object[][]{{8,7,1,1L},{12,6,1,1L},{13,6,1,1L},{22,5,1,1L}});
+
+            sql = "select pk1,pk3,pk4,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk1,pk3,pk4";
+            rs = conn.prepareStatement(sql).executeQuery();
+            assertResultSet(rs, new Object[][]{{1,5,22,1L},{1,6,12,1L},{1,6,13,1L},{1,7,8,1L}});
+
+            sql = "select pk4,pk3,pk1,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk4 desc,pk3 desc,pk1 desc";
+            rs = conn.prepareStatement(sql).executeQuery();
+            assertResultSet(rs, new Object[][]{{22,5,1,1L},{13,6,1,1L},{12,6,1,1L},{8,7,1,1L}});
+
+            sql = "select pk1,pk3,pk4,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk1 desc,pk3 desc,pk4 desc";
+            rs = conn.prepareStatement(sql).executeQuery();
+            assertResultSet(rs, new Object[][]{{1,7,8,1L},{1,6,13,1L},{1,6,12,1L},{1,5,22,1L}});
+        } finally {
+            if(conn != null) {
+                conn.close();
             }
         }
-        assertTrue(!rs.next());
     }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/153b357d/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java
index 3bce9c7..9d6a450 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java
@@ -30,6 +30,7 @@ import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.apache.phoenix.util.TestUtil.assertResultSet;
 
 import java.sql.Connection;
 import java.sql.Date;
@@ -1180,21 +1181,4 @@ public class OrderByIT extends ParallelStatsDisabledIT {
         }
     }
 
-    private void assertResultSet(ResultSet rs,Object[][] rows) throws Exception {
-        for(int rowIndex=0;rowIndex<rows.length;rowIndex++) {
-            assertTrue(rs.next());
-            for(int columnIndex=1;columnIndex<= rows[rowIndex].length;columnIndex++) {
-                Object realValue=rs.getObject(columnIndex);
-                Object expectedValue=rows[rowIndex][columnIndex-1];
-                if(realValue==null) {
-                    assertTrue(expectedValue==null);
-                }
-                else {
-                    assertTrue(realValue.equals(expectedValue));
-                }
-            }
-        }
-        assertTrue(!rs.next());
-    }
-
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/phoenix/blob/153b357d/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryIT.java
index 1da98d2..4a11b41 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryIT.java
@@ -148,7 +148,7 @@ public class SubqueryIT extends BaseJoinIT {
                 "    PARALLEL LEFT-JOIN TABLE 0\n" +
                 "        CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" +
                 "            SERVER FILTER BY FIRST KEY ONLY\n" +
-                "            SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" +
+                "            SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" +
                 "            PARALLEL ANTI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" +
                 "                CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" +
                 "                    SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" +
@@ -156,7 +156,7 @@ public class SubqueryIT extends BaseJoinIT {
                 "    PARALLEL LEFT-JOIN TABLE 1\n" +
                 "        CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" +
                 "            SERVER FILTER BY FIRST KEY ONLY\n" +
-                "            SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" +
+                "            SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" +
                 "            PARALLEL SEMI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" +
                 "                CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" +
                 "                    SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" +
@@ -220,7 +220,7 @@ public class SubqueryIT extends BaseJoinIT {
                 "    PARALLEL LEFT-JOIN TABLE 0\n" +
                 "        CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + " \\[1\\]\n" +
                 "            SERVER FILTER BY FIRST KEY ONLY\n" +
-                "            SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" +
+                "            SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" +
                 "        CLIENT MERGE SORT\n" + 
                 "            PARALLEL ANTI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" +
                 "                CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" +
@@ -229,7 +229,7 @@ public class SubqueryIT extends BaseJoinIT {
                 "    PARALLEL LEFT-JOIN TABLE 1\n" +
                 "        CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + " \\[1\\]\n" +
                 "            SERVER FILTER BY FIRST KEY ONLY\n" +
-                "            SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" +
+                "            SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" +
                 "        CLIENT MERGE SORT\n" + 
                 "            PARALLEL SEMI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" +
                 "                CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" +

http://git-wip-us.apache.org/repos/asf/phoenix/blob/153b357d/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryUsingSortMergeJoinIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryUsingSortMergeJoinIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryUsingSortMergeJoinIT.java
index 9335065..665908f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryUsingSortMergeJoinIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryUsingSortMergeJoinIT.java
@@ -136,7 +136,8 @@ public class SubqueryUsingSortMergeJoinIT extends BaseJoinIT {
                 "    AND\n" +
                 "        CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" +
                 "            SERVER FILTER BY FIRST KEY ONLY\n" +
-                "            SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" +
+                "            SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" +
+                "        CLIENT SORTED BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n"+
                 "            PARALLEL ANTI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" +
                 "                CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" +
                 "                    SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" +
@@ -145,7 +146,8 @@ public class SubqueryUsingSortMergeJoinIT extends BaseJoinIT {
                 "AND\n" +
                 "    CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" +
                 "        SERVER FILTER BY FIRST KEY ONLY\n" +
-                "        SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" +
+                "        SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" +
+                "    CLIENT SORTED BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n"+
                 "        PARALLEL SEMI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" +
                 "            CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" +
                 "                SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" +
@@ -200,8 +202,9 @@ public class SubqueryUsingSortMergeJoinIT extends BaseJoinIT {
                 "    AND\n" +
                 "        CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + " \\[1\\]\n" +
                 "            SERVER FILTER BY FIRST KEY ONLY\n" +
-                "            SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" +
+                "            SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" +
                 "        CLIENT MERGE SORT\n" + 
+                "        CLIENT SORTED BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" +
                 "            PARALLEL ANTI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" +
                 "                CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" +
                 "                    SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" +
@@ -210,8 +213,9 @@ public class SubqueryUsingSortMergeJoinIT extends BaseJoinIT {
                 "AND\n" +
                 "    CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + " \\[1\\]\n" +
                 "        SERVER FILTER BY FIRST KEY ONLY\n" +
-                "        SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" +
+                "        SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" +
                 "    CLIENT MERGE SORT\n" + 
+                "    CLIENT SORTED BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" +
                 "        PARALLEL SEMI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" +
                 "            CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" +
                 "                SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" +

http://git-wip-us.apache.org/repos/asf/phoenix/blob/153b357d/phoenix-core/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java
index a405317..0a9e1bc 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java
@@ -153,9 +153,24 @@ public class GroupByCompiler {
                 // column and use each subsequent one in PK order).
                 isOrderPreserving = tracker.isOrderPreserving();
                 orderPreservingColumnCount = tracker.getOrderPreservingColumnCount();
+                if(isOrderPreserving) {
+                    //reorder the groupby expressions following pk columns
+                    List<Expression> newExpressions = tracker.getExpressionsFromOrderPreservingTrackInfos();
+                    assert newExpressions.size() == expressions.size();
+                    return new GroupBy.GroupByBuilder(this)
+                               .setIsOrderPreserving(isOrderPreserving)
+                               .setOrderPreservingColumnCount(orderPreservingColumnCount)
+                               .setExpressions(newExpressions)
+                               .setKeyExpressions(newExpressions)
+                               .build();
+                }
             }
-            if (isOrderPreserving || isUngroupedAggregate) {
-                return new GroupBy.GroupByBuilder(this).setIsOrderPreserving(isOrderPreserving).setOrderPreservingColumnCount(orderPreservingColumnCount).build();
+
+            if (isUngroupedAggregate) {
+                return new GroupBy.GroupByBuilder(this)
+                           .setIsOrderPreserving(isOrderPreserving)
+                           .setOrderPreservingColumnCount(orderPreservingColumnCount)
+                           .build();
             }
             List<Expression> expressions = Lists.newArrayListWithExpectedSize(this.expressions.size());
             List<Expression> keyExpressions = expressions;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/153b357d/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderPreservingTracker.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderPreservingTracker.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderPreservingTracker.java
index dab2078..d1175f6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderPreservingTracker.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderPreservingTracker.java
@@ -9,6 +9,7 @@
  */
 package org.apache.phoenix.compile;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Iterator;
@@ -49,6 +50,7 @@ public class OrderPreservingTracker {
         public final OrderPreserving orderPreserving;
         public final int pkPosition;
         public final int slotSpan;
+        public Expression expression;
 
         public Info(int pkPosition) {
             this.pkPosition = pkPosition;
@@ -141,6 +143,7 @@ public class OrderPreservingTracker {
                         return;
                     }
                 }
+                info.expression = node;
                 orderPreservingInfos.add(info);
             }
         }
@@ -153,6 +156,16 @@ public class OrderPreservingTracker {
         return orderPreservingColumnCount;
     }
 
+    public List<Expression> getExpressionsFromOrderPreservingTrackInfos() {
+        assert isOrderPreserving;
+        assert (this.orderPreservingInfos != null && this.orderPreservingInfos.size() > 0);
+        List<Expression> newExpressions = new ArrayList<Expression>(this.orderPreservingInfos.size());
+        for(Info trackInfo : this.orderPreservingInfos) {
+            newExpressions.add(trackInfo.expression);
+        }
+        return newExpressions;
+    }
+
     public boolean isOrderPreserving() {
         if (!isOrderPreserving) {
             return false;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/153b357d/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
index bac4ee8..07bd3a9 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
@@ -98,6 +98,7 @@ import org.apache.phoenix.util.PropertiesUtil;
 import org.apache.phoenix.util.QueryUtil;
 import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -4850,4 +4851,113 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest {
             return Collections.emptyList();
         }
     }
+
+    @Test
+    public void testGroupByOrderMatchPkColumnOrder4690() throws Exception{
+        this.doTestGroupByOrderMatchPkColumnOrderBug4690(false, false);
+        this.doTestGroupByOrderMatchPkColumnOrderBug4690(false, true);
+        this.doTestGroupByOrderMatchPkColumnOrderBug4690(true, false);
+        this.doTestGroupByOrderMatchPkColumnOrderBug4690(true, true);
+    }
+
+    private void doTestGroupByOrderMatchPkColumnOrderBug4690(boolean desc ,boolean salted) throws Exception {
+        Connection conn = null;
+        try {
+            conn = DriverManager.getConnection(getUrl());
+            String tableName = generateUniqueName();
+            String sql = "create table " + tableName + "( "+
+                    " pk1 integer not null , " +
+                    " pk2 integer not null, " +
+                    " pk3 integer not null," +
+                    " pk4 integer not null,"+
+                    " v integer, " +
+                    " CONSTRAINT TEST_PK PRIMARY KEY ( "+
+                       "pk1 "+(desc ? "desc" : "")+", "+
+                       "pk2 "+(desc ? "desc" : "")+", "+
+                       "pk3 "+(desc ? "desc" : "")+", "+
+                       "pk4 "+(desc ? "desc" : "")+
+                    " )) "+(salted ? "SALT_BUCKETS =4" : "split on(2)");
+            conn.createStatement().execute(sql);
+
+            sql = "select pk2,pk1,count(v) from " + tableName + " group by pk2,pk1 order by pk2,pk1";
+            QueryPlan queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+            assertTrue(queryPlan.getGroupBy().isOrderPreserving());
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().size() ==2);
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().get(0).toString().equals("PK2"));
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().get(1).toString().equals("PK1"));
+
+            sql = "select pk1,pk2,count(v) from " + tableName + " group by pk2,pk1 order by pk1,pk2";
+            queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+            assertTrue(queryPlan.getGroupBy().isOrderPreserving());
+            assertTrue(queryPlan.getOrderBy() == (!desc ? OrderBy.FWD_ROW_KEY_ORDER_BY : OrderBy.REV_ROW_KEY_ORDER_BY));
+
+            sql = "select pk2,pk1,count(v) from " + tableName + " group by pk2,pk1 order by pk2 desc,pk1 desc";
+            queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+            assertTrue(queryPlan.getGroupBy().isOrderPreserving());
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().size() ==2);
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().get(0).toString().equals("PK2 DESC"));
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().get(1).toString().equals("PK1 DESC"));
+
+            sql = "select pk1,pk2,count(v) from " + tableName + " group by pk2,pk1 order by pk1 desc,pk2 desc";
+            queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+            assertTrue(queryPlan.getGroupBy().isOrderPreserving());
+            assertTrue(queryPlan.getOrderBy() == (!desc ? OrderBy.REV_ROW_KEY_ORDER_BY : OrderBy.FWD_ROW_KEY_ORDER_BY));
+
+
+            sql = "select pk3,pk2,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk3,pk2";
+            queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+            assertTrue(queryPlan.getGroupBy().isOrderPreserving());
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().size() == 2);
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().get(0).toString().equals("PK3"));
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().get(1).toString().equals("PK2"));
+
+            sql = "select pk2,pk3,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk2,pk3";
+            queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+            assertTrue(queryPlan.getGroupBy().isOrderPreserving());
+            assertTrue(queryPlan.getOrderBy() == (!desc ? OrderBy.FWD_ROW_KEY_ORDER_BY : OrderBy.REV_ROW_KEY_ORDER_BY));
+
+            sql = "select pk3,pk2,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk3 desc,pk2 desc";
+            queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+            assertTrue(queryPlan.getGroupBy().isOrderPreserving());
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().size() == 2);
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().get(0).toString().equals("PK3 DESC"));
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().get(1).toString().equals("PK2 DESC"));
+
+            sql = "select pk2,pk3,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk2 desc,pk3 desc";
+            queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+            assertTrue(queryPlan.getGroupBy().isOrderPreserving());
+            assertTrue(queryPlan.getOrderBy() == (!desc ? OrderBy.REV_ROW_KEY_ORDER_BY : OrderBy.FWD_ROW_KEY_ORDER_BY));
+
+
+            sql = "select pk4,pk3,pk1,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk4,pk3,pk1";
+            queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+            assertTrue(queryPlan.getGroupBy().isOrderPreserving());
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().size() == 3);
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().get(0).toString().equals("PK4"));
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().get(1).toString().equals("PK3"));
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().get(2).toString().equals("PK1"));
+
+            sql = "select pk1,pk3,pk4,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk1,pk3,pk4";
+            queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+            assertTrue(queryPlan.getGroupBy().isOrderPreserving());
+            assertTrue(queryPlan.getOrderBy() == (!desc ? OrderBy.FWD_ROW_KEY_ORDER_BY : OrderBy.REV_ROW_KEY_ORDER_BY));
+
+            sql = "select pk4,pk3,pk1,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk4 desc,pk3 desc,pk1 desc";
+            queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+            assertTrue(queryPlan.getGroupBy().isOrderPreserving());
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().size() == 3);
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().get(0).toString().equals("PK4 DESC"));
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().get(1).toString().equals("PK3 DESC"));
+            assertTrue(queryPlan.getOrderBy().getOrderByExpressions().get(2).toString().equals("PK1 DESC"));
+
+            sql = "select pk1,pk3,pk4,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk1 desc,pk3 desc,pk4 desc";
+            queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+            assertTrue(queryPlan.getGroupBy().isOrderPreserving());
+            assertTrue(queryPlan.getOrderBy() == (!desc ? OrderBy.REV_ROW_KEY_ORDER_BY : OrderBy.FWD_ROW_KEY_ORDER_BY));
+        } finally {
+            if(conn != null) {
+                conn.close();
+            }
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/153b357d/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index a06fd69..277e257 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -1043,4 +1043,24 @@ public class TestUtil {
         return queryPlan;
     }
 
+    public static void assertResultSet(ResultSet rs,Object[][] rows) throws Exception {
+        for(int rowIndex=0; rowIndex < rows.length; rowIndex++) {
+            assertTrue("rowIndex:["+rowIndex+"] rs.next error!",rs.next());
+            for(int columnIndex = 1; columnIndex <= rows[rowIndex].length; columnIndex++) {
+                Object realValue = rs.getObject(columnIndex);
+                Object expectedValue = rows[rowIndex][columnIndex-1];
+                if(realValue == null) {
+                    assertNull("rowIndex:["+rowIndex+"],columnIndex:["+columnIndex+"]",expectedValue);
+                }
+                else {
+                    assertEquals("rowIndex:["+rowIndex+"],columnIndex:["+columnIndex+"],realValue:["+
+                            realValue+"],expectedValue:["+expectedValue+"]",
+                            expectedValue,
+                            realValue
+                            );
+                }
+            }
+        }
+        assertTrue(!rs.next());
+    }
 }


[36/50] [abbrv] phoenix git commit: PHOENIX-4699 Stop scan after finding first child of table during drop

Posted by td...@apache.org.
PHOENIX-4699 Stop scan after finding first child of table during drop


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/041e7b5a
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/041e7b5a
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/041e7b5a

Branch: refs/heads/system-catalog
Commit: 041e7b5a05213710fc2af0644145d8247cb02191
Parents: 8f1cef8
Author: James Taylor <jt...@salesforce.com>
Authored: Thu Apr 19 14:39:27 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Thu Apr 26 09:55:27 2018 -0700

----------------------------------------------------------------------
 .../coprocessor/MetaDataEndpointImpl.java       | 44 +++++++++++---------
 1 file changed, 25 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/041e7b5a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 4c72c2d..c28ad3c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -122,8 +122,10 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.FilterList;
 import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
+import org.apache.hadoop.hbase.filter.PageFilter;
 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.ipc.RpcServer.Call;
@@ -275,6 +277,10 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
     public static final String ROW_KEY_ORDER_OPTIMIZABLE = "ROW_KEY_ORDER_OPTIMIZABLE";
     public static final byte[] ROW_KEY_ORDER_OPTIMIZABLE_BYTES = Bytes.toBytes(ROW_KEY_ORDER_OPTIMIZABLE);
 
+    private static final byte[] CHILD_TABLE_BYTES = new byte[] {PTable.LinkType.CHILD_TABLE.getSerializedValue()};
+    private static final byte[] PHYSICAL_TABLE_BYTES =
+            new byte[] { PTable.LinkType.PHYSICAL_TABLE.getSerializedValue() };
+
     // KeyValues for Table
     private static final KeyValue TABLE_TYPE_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, TABLE_TYPE_BYTES);
     private static final KeyValue TABLE_SEQ_NUM_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, TABLE_SEQ_NUM_BYTES);
@@ -1527,7 +1533,6 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                     for (PTable index : parentTable.getIndexes()) {
                         indexes.add(TableName.valueOf(index.getPhysicalName().getBytes()));
                     }
-
                 } else {
                     // Mapped View
                     cParentPhysicalName = SchemaUtil.getTableNameAsBytes(schemaName, tableName);
@@ -1810,13 +1815,10 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
     private boolean execeededIndexQuota(PTableType tableType, PTable parentTable) {
         return PTableType.INDEX == tableType && parentTable.getIndexes().size() >= maxIndexesPerTable;
     }
-
-    private static final byte[] CHILD_TABLE_BYTES = new byte[] {PTable.LinkType.CHILD_TABLE.getSerializedValue()};
-
     
     private void findAllChildViews(Region region, byte[] tenantId, PTable table,
             TableViewFinder result, long clientTimeStamp, int clientVersion) throws IOException, SQLException {
-        TableViewFinder currResult = findChildViews(region, tenantId, table, clientVersion);
+        TableViewFinder currResult = findChildViews(region, tenantId, table, clientVersion, false);
         result.addResult(currResult);
         for (ViewInfo viewInfo : currResult.getViewInfoList()) {
             byte[] viewtenantId = viewInfo.getTenantId();
@@ -1829,9 +1831,9 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
         }
     }
         
-    // TODO remove this in 4.13 release 
-    @Deprecated
-    private TableViewFinder findChildViews_deprecated(Region region, byte[] tenantId, PTable table, byte[] linkTypeBytes) throws IOException {
+    // TODO use child link instead once splittable system catalog (PHOENIX-3534) is implemented
+    // and we have a separate table for links.
+    private TableViewFinder findChildViews_deprecated(Region region, byte[] tenantId, PTable table, byte[] linkTypeBytes, boolean stopAfterFirst) throws IOException {
         byte[] schemaName = table.getSchemaName().getBytes();
         byte[] tableName = table.getTableName().getBytes();
         boolean isMultiTenant = table.isMultiTenant();
@@ -1854,7 +1856,11 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                 .getPhysicalHBaseTableName(schemaName, tableName, table.isNamespaceMapped())
                 .getBytes());
         SuffixFilter rowFilter = new SuffixFilter(suffix);
-        FilterList filter = new FilterList(linkFilter,tableTypeFilter,rowFilter);
+        List<Filter> filters = Lists.<Filter>newArrayList(linkFilter,tableTypeFilter,rowFilter);
+        if (stopAfterFirst) {
+            filters.add(new PageFilter(1));
+        }
+        FilterList filter = new FilterList(filters);
         scan.setFilter(filter);
         scan.addColumn(TABLE_FAMILY_BYTES, LINK_TYPE_BYTES);
         scan.addColumn(TABLE_FAMILY_BYTES, TABLE_TYPE_BYTES);
@@ -1897,15 +1903,19 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
         }
     }
     
-    private TableViewFinder findChildViews_4_11(Region region, byte[] tenantId, byte[] schemaName, byte[] tableName) throws IOException {
+    private TableViewFinder findChildViews_4_11(Region region, byte[] tenantId, byte[] schemaName, byte[] tableName, boolean stopAfterFirst) throws IOException {
         Scan scan = new Scan();
         byte[] startRow = SchemaUtil.getTableKey(tenantId, schemaName, tableName);
         byte[] stopRow = ByteUtil.nextKey(startRow);
         scan.setStartRow(startRow);
         scan.setStopRow(stopRow);
         SingleColumnValueFilter linkFilter = new SingleColumnValueFilter(TABLE_FAMILY_BYTES, LINK_TYPE_BYTES, CompareOp.EQUAL, CHILD_TABLE_BYTES);
+        Filter filter = linkFilter;
         linkFilter.setFilterIfMissing(true);
-        scan.setFilter(linkFilter);
+        if (stopAfterFirst) {
+            filter = new FilterList(linkFilter, new PageFilter(1));
+        }
+        scan.setFilter(filter);
         scan.addColumn(TABLE_FAMILY_BYTES, LINK_TYPE_BYTES);
         scan.addColumn(TABLE_FAMILY_BYTES, PARENT_TENANT_ID_BYTES);
         
@@ -1945,11 +1955,8 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
             }
         }
     }
-    
-    private static final byte[] PHYSICAL_TABLE_BYTES =
-            new byte[] { PTable.LinkType.PHYSICAL_TABLE.getSerializedValue() };
 
-    private TableViewFinder findChildViews(Region region, byte[] tenantId, PTable table, int clientVersion)
+    private TableViewFinder findChildViews(Region region, byte[] tenantId, PTable table, int clientVersion, boolean stopAfterFirst)
             throws IOException, SQLException {
         byte[] tableKey =
                 SchemaUtil.getTableKey(ByteUtil.EMPTY_BYTE_ARRAY,
@@ -1960,10 +1967,10 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                 loadTable(env, tableKey, cacheKey, MIN_SYSTEM_TABLE_TIMESTAMP,
                     HConstants.LATEST_TIMESTAMP, clientVersion);
         if (systemCatalog.getTimeStamp() < MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0) {
-            return findChildViews_deprecated(region, tenantId, table, PHYSICAL_TABLE_BYTES);
+            return findChildViews_deprecated(region, tenantId, table, PHYSICAL_TABLE_BYTES, stopAfterFirst);
         } else {
             return findChildViews_4_11(region, tenantId, table.getSchemaName().getBytes(),
-                table.getTableName().getBytes());
+                table.getTableName().getBytes(), stopAfterFirst);
         }
     }
     
@@ -2131,7 +2138,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
 
             if (tableType == PTableType.TABLE || tableType == PTableType.SYSTEM) {
                 // Handle any child views that exist
-                TableViewFinder tableViewFinderResult = findChildViews(region, tenantId, table, clientVersion);
+                TableViewFinder tableViewFinderResult = findChildViews(region, tenantId, table, clientVersion, !isCascade);
                 if (tableViewFinderResult.hasViews()) {
                     if (isCascade) {
                         if (tableViewFinderResult.allViewsInMultipleRegions()) {
@@ -2541,7 +2548,6 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
             // lock the rows corresponding to views so that no other thread can modify the view meta-data
             RowLock viewRowLock = acquireLock(region, viewKey, locks);
             PTable view = doGetTable(viewKey, clientTimeStamp, viewRowLock, clientVersion);
-            
             ColumnOrdinalPositionUpdateList ordinalPositionList = new ColumnOrdinalPositionUpdateList();
             List<PColumn> viewPkCols = new ArrayList<>(view.getPKColumns());
             boolean addingExistingPkCol = false;


[22/50] [abbrv] phoenix git commit: PHOENIX-4496 Fix RowValueConstructorIT and IndexMetadataIT

Posted by td...@apache.org.
PHOENIX-4496 Fix RowValueConstructorIT and IndexMetadataIT


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/db0e1eda
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/db0e1eda
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/db0e1eda

Branch: refs/heads/system-catalog
Commit: db0e1eda6025d640b55eb30ece2047761cdc8ca4
Parents: 430e802
Author: Ankit Singhal <an...@gmail.com>
Authored: Sat Apr 14 09:54:11 2018 +0530
Committer: Ankit Singhal <an...@gmail.com>
Committed: Sat Apr 14 09:54:11 2018 +0530

----------------------------------------------------------------------
 .../phoenix/hbase/index/scanner/ScannerBuilder.java | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/db0e1eda/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
index ad09c0c..703fcd2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.FamilyFilter;
 import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
 import org.apache.hadoop.hbase.filter.FilterList;
 import org.apache.hadoop.hbase.filter.QualifierFilter;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -98,6 +99,21 @@ public class ScannerBuilder {
       }
       columnFilters.addFilter(columnFilter);
     }
+    
+    if(columns.isEmpty()){
+        columnFilters.addFilter(new FilterBase() {
+            
+            @Override
+            public boolean filterAllRemaining() throws IOException {
+                return true;
+            }
+
+            @Override
+            public ReturnCode filterKeyValue(Cell v) throws IOException {
+                return Filter.ReturnCode.INCLUDE;
+            }
+        });
+    }
     return columnFilters;
   }
 


[50/50] [abbrv] phoenix git commit: Merge branch 'PHOENIX-3534' into system-catalog

Posted by td...@apache.org.
Merge branch 'PHOENIX-3534' into system-catalog


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/f7d87ce2
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/f7d87ce2
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/f7d87ce2

Branch: refs/heads/system-catalog
Commit: f7d87ce2fec91bf5eb7d4d06f8f1d35579669440
Parents: 15d4dac 7420ae3
Author: Thomas D'Silva <td...@apache.org>
Authored: Wed May 2 20:32:53 2018 -0700
Committer: Thomas D'Silva <td...@apache.org>
Committed: Wed May 2 20:32:53 2018 -0700

----------------------------------------------------------------------
 bin/performance.py                              |    4 +-
 bin/phoenix_utils.py                            |    2 +-
 dev/make_rc.sh                                  |   15 +-
 phoenix-assembly/pom.xml                        |    2 +-
 .../src/build/components/all-common-files.xml   |    5 +
 phoenix-assembly/src/build/src.xml              |    7 +-
 phoenix-client/pom.xml                          |    2 +-
 phoenix-core/pom.xml                            |   16 +-
 ...ReplayWithIndexWritesAndCompressedWALIT.java |    2 +-
 .../org/apache/phoenix/end2end/AggregateIT.java |  130 +-
 .../apache/phoenix/end2end/AlterTableIT.java    |   14 +-
 .../phoenix/end2end/AlterTableWithViewsIT.java  |   28 +-
 .../phoenix/end2end/AppendOnlySchemaIT.java     |    2 +-
 .../phoenix/end2end/ArrayAppendFunctionIT.java  |   43 +-
 .../phoenix/end2end/ArrayRemoveFunctionIT.java  |  383 ++++
 .../phoenix/end2end/BasePermissionsIT.java      |    4 +
 .../phoenix/end2end/ChangePermissionsIT.java    |   34 +-
 .../phoenix/end2end/CollationKeyFunctionIT.java |   37 +-
 .../end2end/ColumnProjectionOptimizationIT.java |   14 +-
 .../ConnectionQueryServicesTestImpl.java        |   34 +-
 .../phoenix/end2end/CostBasedDecisionIT.java    |  420 +++--
 .../apache/phoenix/end2end/CreateTableIT.java   |    5 +-
 .../org/apache/phoenix/end2end/DeleteIT.java    |  101 ++
 .../org/apache/phoenix/end2end/DropTableIT.java |   43 +
 .../apache/phoenix/end2end/DynamicColumnIT.java |   14 +-
 .../apache/phoenix/end2end/DynamicFamilyIT.java |   26 +-
 .../end2end/ExplainPlanWithStatsEnabledIT.java  |  383 ++--
 .../apache/phoenix/end2end/IndexExtendedIT.java |    6 +-
 .../org/apache/phoenix/end2end/IndexToolIT.java |    6 +-
 .../phoenix/end2end/LocalIndexSplitMergeIT.java |   59 +
 .../org/apache/phoenix/end2end/MapReduceIT.java |   69 +-
 .../phoenix/end2end/MappingTableDataTypeIT.java |    6 +-
 .../MigrateSystemTablesToSystemNamespaceIT.java |   49 +-
 .../phoenix/end2end/MultiCfQueryExecIT.java     |   47 +
 .../end2end/NamespaceSchemaMappingIT.java       |   19 +-
 .../phoenix/end2end/NativeHBaseTypesIT.java     |   30 +-
 .../org/apache/phoenix/end2end/OrderByIT.java   |  129 +-
 .../phoenix/end2end/ParallelStatsEnabledIT.java |    1 +
 .../PartialScannerResultsDisabledIT.java        |    2 +-
 .../apache/phoenix/end2end/PhoenixDriverIT.java |   99 +-
 .../end2end/QueryDatabaseMetaDataIT.java        |   39 +-
 .../apache/phoenix/end2end/QueryLoggerIT.java   |  358 ++++
 .../apache/phoenix/end2end/ReverseScanIT.java   |   88 +-
 .../phoenix/end2end/SortMergeJoinMoreIT.java    |  116 +-
 .../phoenix/end2end/SpillableGroupByIT.java     |   74 +-
 .../SystemCatalogCreationOnConnectionIT.java    |  638 +++++++
 .../end2end/SystemTablePermissionsIT.java       |    2 +-
 .../end2end/TenantSpecificTablesDDLIT.java      |    2 +
 .../UngroupedAggregateRegionObserverIT.java     |  171 --
 .../org/apache/phoenix/end2end/UpgradeIT.java   |   14 +-
 .../end2end/UpsertSelectAutoCommitIT.java       |   28 +
 .../phoenix/end2end/UserDefinedFunctionsIT.java |  165 +-
 .../phoenix/end2end/index/BaseIndexIT.java      |   22 +
 .../index/ChildViewsUseParentViewIndexIT.java   |   10 +-
 .../phoenix/end2end/index/ImmutableIndexIT.java |    3 +
 .../end2end/index/MutableIndexFailureIT.java    |   27 +-
 .../phoenix/end2end/index/MutableIndexIT.java   |   95 +
 .../end2end/index/MutableIndexRebuilderIT.java  |  143 ++
 .../end2end/index/PartialIndexRebuilderIT.java  |  130 +-
 .../phoenix/end2end/index/ViewIndexIT.java      |   45 +-
 .../end2end/index/txn/TxWriteFailureIT.java     |    6 +-
 .../apache/phoenix/end2end/join/BaseJoinIT.java |    2 +
 .../phoenix/end2end/join/HashJoinMoreIT.java    |    4 +-
 .../apache/phoenix/end2end/join/SubqueryIT.java |   49 +-
 .../join/SubqueryUsingSortMergeJoinIT.java      |   12 +-
 .../phoenix/monitoring/PhoenixMetricsIT.java    |   24 +-
 .../phoenix/tx/FlappingTransactionIT.java       |   11 +-
 .../phoenix/tx/ParameterizedTransactionIT.java  |   55 +-
 .../org/apache/phoenix/tx/TransactionIT.java    |   12 +
 .../org/apache/phoenix/tx/TxCheckpointIT.java   |    5 +-
 .../hadoop/hbase/ipc/PhoenixRpcScheduler.java   |   30 +
 .../controller/ClientRpcControllerFactory.java  |   18 +-
 .../ipc/controller/IndexRpcController.java      |    8 +-
 ...erRegionServerIndexRpcControllerFactory.java |   18 +-
 ...egionServerMetadataRpcControllerFactory.java |   17 +-
 .../ipc/controller/MetadataRpcController.java   |    8 +-
 .../IndexHalfStoreFileReaderGenerator.java      |   30 +-
 .../LocalIndexStoreFileScanner.java             |    4 +-
 .../hadoop/hbase/regionserver/ScanInfoUtil.java |   12 +
 .../org/apache/phoenix/cache/GlobalCache.java   |    6 +-
 .../org/apache/phoenix/cache/HashCache.java     |    1 +
 .../phoenix/cache/IndexMetaDataCache.java       |    7 +
 .../apache/phoenix/cache/ServerCacheClient.java |    7 +-
 .../org/apache/phoenix/cache/TenantCache.java   |    2 +-
 .../apache/phoenix/cache/TenantCacheImpl.java   |    4 +-
 .../apache/phoenix/compile/DeleteCompiler.java  |  124 +-
 .../org/apache/phoenix/compile/ExplainPlan.java |   10 +
 .../apache/phoenix/compile/FromCompiler.java    |    2 +-
 .../apache/phoenix/compile/GroupByCompiler.java |   19 +-
 .../apache/phoenix/compile/JoinCompiler.java    |  324 ++--
 .../phoenix/compile/ListJarsQueryPlan.java      |    8 +-
 .../apache/phoenix/compile/OrderByCompiler.java |   11 +-
 .../phoenix/compile/OrderPreservingTracker.java |   13 +
 .../apache/phoenix/compile/PostDDLCompiler.java |    2 +-
 .../apache/phoenix/compile/QueryCompiler.java   |  498 +++---
 .../org/apache/phoenix/compile/QueryPlan.java   |    2 +
 .../org/apache/phoenix/compile/ScanRanges.java  |   17 +-
 .../apache/phoenix/compile/SequenceManager.java |    2 +-
 .../phoenix/compile/StatementContext.java       |   10 +
 .../phoenix/compile/SubselectRewriter.java      |    5 +
 .../apache/phoenix/compile/TraceQueryPlan.java  |    6 +
 .../compile/TupleProjectionCompiler.java        |   44 +-
 .../apache/phoenix/compile/UnionCompiler.java   |    6 +-
 .../apache/phoenix/compile/UpsertCompiler.java  |   47 +-
 .../apache/phoenix/compile/WhereOptimizer.java  |   23 +-
 .../coprocessor/BaseScannerRegionObserver.java  |   15 +-
 .../DelegateRegionCoprocessorEnvironment.java   |   15 +-
 .../coprocessor/DelegateRegionObserver.java     |   32 +-
 .../GroupedAggregateRegionObserver.java         |    4 +
 .../coprocessor/MetaDataEndpointImpl.java       |  188 +-
 .../phoenix/coprocessor/MetaDataProtocol.java   |   15 +-
 .../coprocessor/MetaDataRegionObserver.java     |   21 +-
 .../coprocessor/PhoenixAccessController.java    |  102 +-
 .../PhoenixMetaDataCoprocessorHost.java         |    6 +
 .../PhoenixTransactionalProcessor.java          |   28 -
 .../phoenix/coprocessor/ScanRegionObserver.java |    4 +-
 .../coprocessor/ServerCachingEndpointImpl.java  |    4 +-
 .../coprocessor/ServerCachingProtocol.java      |    2 +-
 .../TephraTransactionalProcessor.java           |   29 +
 .../UngroupedAggregateRegionObserver.java       |  311 ++--
 .../coprocessor/generated/MetaDataProtos.java   |  183 +-
 .../coprocessor/generated/PTableProtos.java     |  110 +-
 .../generated/ServerCachingProtos.java          |  117 +-
 .../phoenix/exception/SQLExceptionCode.java     |   17 +-
 .../exception/UpgradeRequiredException.java     |   13 +-
 .../apache/phoenix/execute/AggregatePlan.java   |   53 +-
 .../apache/phoenix/execute/BaseQueryPlan.java   |   21 +-
 .../phoenix/execute/ClientAggregatePlan.java    |   46 +-
 .../phoenix/execute/ClientProcessingPlan.java   |    4 +
 .../apache/phoenix/execute/ClientScanPlan.java  |   22 +-
 .../apache/phoenix/execute/CorrelatePlan.java   |   26 +-
 .../apache/phoenix/execute/CursorFetchPlan.java |    6 +
 .../apache/phoenix/execute/DelegateHTable.java  |   20 +
 .../apache/phoenix/execute/HashJoinPlan.java    |  128 +-
 .../execute/LiteralResultIterationPlan.java     |    8 +-
 .../apache/phoenix/execute/MutationState.java   |  219 ++-
 .../PhoenixTxIndexMutationGenerator.java        |  443 +++++
 .../org/apache/phoenix/execute/ScanPlan.java    |   36 +-
 .../phoenix/execute/SortMergeJoinPlan.java      |   20 +-
 .../phoenix/execute/TupleProjectionPlan.java    |    6 +
 .../org/apache/phoenix/execute/UnionPlan.java   |   12 +-
 .../apache/phoenix/execute/UnnestArrayPlan.java |    6 +
 .../execute/visitor/AvgRowWidthVisitor.java     |  205 +++
 .../execute/visitor/ByteCountVisitor.java       |  125 ++
 .../execute/visitor/QueryPlanVisitor.java       |   46 +
 .../execute/visitor/RowCountVisitor.java        |  335 ++++
 .../expression/BaseSingleExpression.java        |    5 +
 .../phoenix/expression/ExpressionType.java      |  119 +-
 .../expression/aggregator/Aggregators.java      |    3 +-
 .../aggregator/ClientAggregators.java           |    3 +-
 .../DistinctValueWithCountServerAggregator.java |   20 +-
 .../aggregator/ServerAggregators.java           |    6 +-
 .../function/ArrayAppendFunction.java           |   11 +-
 .../function/ArrayConcatFunction.java           |    3 +-
 .../function/ArrayModifierFunction.java         |   55 +-
 .../function/ArrayPrependFunction.java          |    9 +-
 .../function/ArrayRemoveFunction.java           |   88 +
 .../function/CollationKeyFunction.java          |   24 +-
 .../expression/function/LowerFunction.java      |   28 +-
 .../expression/function/ScalarFunction.java     |   15 +-
 .../TransactionProviderNameFunction.java        |   81 +
 .../expression/function/UDFExpression.java      |   39 +-
 .../expression/function/UpperFunction.java      |   29 +-
 .../org/apache/phoenix/hbase/index/Indexer.java |   25 +-
 .../hbase/index/builder/BaseIndexBuilder.java   |    4 +-
 .../hbase/index/builder/BaseIndexCodec.java     |    7 -
 .../hbase/index/builder/IndexBuildManager.java  |    4 +
 .../phoenix/hbase/index/covered/IndexCodec.java |   14 +-
 .../hbase/index/covered/IndexMetaData.java      |   12 +-
 .../hbase/index/covered/LocalTableState.java    |   16 +-
 .../hbase/index/covered/NonTxIndexBuilder.java  |    2 +-
 .../phoenix/hbase/index/covered/TableState.java |   15 -
 .../index/exception/IndexWriteException.java    |   52 +-
 .../MultiIndexWriteFailureException.java        |   39 +-
 .../SingleIndexWriteFailureException.java       |   32 +-
 .../hbase/index/scanner/ScannerBuilder.java     |   16 +
 .../hbase/index/write/IndexCommitter.java       |    2 +-
 .../phoenix/hbase/index/write/IndexWriter.java  |   21 +-
 .../hbase/index/write/IndexWriterUtils.java     |   87 +-
 .../write/ParallelWriterIndexCommitter.java     |   19 +-
 .../hbase/index/write/RecoveryIndexWriter.java  |    4 +-
 .../TrackingParallelWriterIndexCommitter.java   |   20 +-
 .../apache/phoenix/index/IndexMaintainer.java   |   39 +-
 .../index/IndexMetaDataCacheFactory.java        |    9 +-
 .../phoenix/index/PhoenixIndexBuilder.java      |   21 +-
 .../apache/phoenix/index/PhoenixIndexCodec.java |   36 +-
 .../index/PhoenixIndexFailurePolicy.java        |  207 ++-
 .../phoenix/index/PhoenixIndexMetaData.java     |   74 +-
 .../index/PhoenixIndexMetaDataBuilder.java      |  107 ++
 .../index/PhoenixTransactionalIndexer.java      |  469 +----
 .../phoenix/iterate/BaseResultIterators.java    |  480 ++++-
 .../apache/phoenix/iterate/ExplainTable.java    |    3 +-
 .../NonAggregateRegionScannerFactory.java       |   61 +-
 .../phoenix/iterate/ParallelIterators.java      |   16 +-
 .../phoenix/iterate/RegionScannerFactory.java   |    4 +-
 .../phoenix/iterate/ScanningResultIterator.java |   26 +-
 .../apache/phoenix/iterate/SerialIterators.java |   17 +-
 .../apache/phoenix/iterate/SnapshotScanner.java |   12 +-
 .../phoenix/iterate/TableResultIterator.java    |    7 +-
 .../apache/phoenix/jdbc/PhoenixConnection.java  |   19 +-
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |   36 +-
 .../phoenix/jdbc/PhoenixPreparedStatement.java  |   11 +-
 .../apache/phoenix/jdbc/PhoenixResultSet.java   |   38 +
 .../apache/phoenix/jdbc/PhoenixStatement.java   |   85 +-
 .../apache/phoenix/join/HashCacheFactory.java   |   34 +-
 .../java/org/apache/phoenix/log/LogLevel.java   |   22 +
 .../java/org/apache/phoenix/log/LogWriter.java  |   51 +
 .../log/QueryLogDetailsEventHandler.java        |   63 +
 .../org/apache/phoenix/log/QueryLogInfo.java    |   87 +
 .../org/apache/phoenix/log/QueryLogState.java   |   22 +
 .../org/apache/phoenix/log/QueryLogger.java     |  145 ++
 .../log/QueryLoggerDefaultExceptionHandler.java |   51 +
 .../phoenix/log/QueryLoggerDisruptor.java       |  117 ++
 .../org/apache/phoenix/log/QueryLoggerUtil.java |   62 +
 .../org/apache/phoenix/log/RingBufferEvent.java |   93 +
 .../phoenix/log/RingBufferEventTranslator.java  |   53 +
 .../org/apache/phoenix/log/TableLogWriter.java  |  125 ++
 .../phoenix/mapreduce/PhoenixInputFormat.java   |   51 +-
 .../index/PhoenixIndexPartialBuildMapper.java   |    5 +
 .../util/PhoenixConfigurationUtil.java          |   63 +-
 .../mapreduce/util/PhoenixMapReduceUtil.java    |    5 +-
 .../phoenix/memory/ChildMemoryManager.java      |    7 +-
 .../phoenix/memory/GlobalMemoryManager.java     |   19 +-
 .../phoenix/monitoring/GlobalClientMetrics.java |   27 +-
 .../phoenix/monitoring/ReadMetricQueue.java     |   44 +-
 .../phoenix/monitoring/ScanMetricsHolder.java   |   48 +-
 .../GenSubqueryParamValuesRewriter.java         |  153 ++
 .../apache/phoenix/optimize/QueryOptimizer.java |  212 ++-
 .../phoenix/parse/ArrayModifierParseNode.java   |   36 +
 .../apache/phoenix/parse/FunctionParseNode.java |   12 +-
 .../java/org/apache/phoenix/parse/HintNode.java |    4 +
 .../org/apache/phoenix/parse/ParseNode.java     |   26 +-
 .../phoenix/query/ConnectionQueryServices.java  |   12 +-
 .../query/ConnectionQueryServicesImpl.java      | 1016 ++++++-----
 .../query/ConnectionlessQueryServicesImpl.java  |   76 +-
 .../query/DelegateConnectionQueryServices.java  |   23 +-
 .../apache/phoenix/query/QueryConstants.java    |  429 ++---
 .../org/apache/phoenix/query/QueryServices.java |   16 +-
 .../phoenix/query/QueryServicesOptions.java     |   29 +-
 .../apache/phoenix/schema/DelegateTable.java    |    8 +-
 .../apache/phoenix/schema/MetaDataClient.java   |  357 ++--
 .../org/apache/phoenix/schema/PColumnImpl.java  |    6 +-
 .../org/apache/phoenix/schema/PIndexState.java  |    7 +-
 .../apache/phoenix/schema/PMetaDataImpl.java    |    2 +-
 .../java/org/apache/phoenix/schema/PTable.java  |    2 +
 .../org/apache/phoenix/schema/PTableImpl.java   |   96 +-
 .../phoenix/schema/RowKeyValueAccessor.java     |    2 +-
 .../SplitOnLeadingVarCharColumnsPolicy.java     |   43 +
 .../schema/SystemFunctionSplitPolicy.java       |   27 +
 .../phoenix/schema/SystemStatsSplitPolicy.java  |   27 +
 .../apache/phoenix/schema/TableProperty.java    |  288 +--
 .../schema/TablesNotInSyncException.java        |   17 +
 .../phoenix/schema/stats/StatisticsUtil.java    |    6 +
 .../phoenix/schema/stats/StatisticsWriter.java  |    9 +-
 .../transaction/OmidTransactionContext.java     |   57 +-
 .../transaction/OmidTransactionProvider.java    |   84 +
 .../transaction/OmidTransactionTable.java       |  364 ----
 .../transaction/PhoenixTransactionClient.java   |   23 +
 .../transaction/PhoenixTransactionContext.java  |  169 +-
 .../transaction/PhoenixTransactionProvider.java |   51 +
 .../transaction/PhoenixTransactionService.java  |   24 +
 .../transaction/PhoenixTransactionalTable.java  |  149 --
 .../transaction/TephraTransactionContext.java   |  205 +--
 .../transaction/TephraTransactionProvider.java  |  197 +++
 .../transaction/TephraTransactionTable.java     |  350 ----
 .../phoenix/transaction/TransactionFactory.java |  139 +-
 .../java/org/apache/phoenix/util/CostUtil.java  |   61 +-
 .../org/apache/phoenix/util/ExpressionUtil.java |    2 +-
 .../java/org/apache/phoenix/util/IndexUtil.java |    6 +-
 .../org/apache/phoenix/util/KeyValueUtil.java   |   12 +
 .../org/apache/phoenix/util/PhoenixRuntime.java |   27 +-
 .../java/org/apache/phoenix/util/QueryUtil.java |   14 +-
 .../java/org/apache/phoenix/util/ScanUtil.java  |   29 +-
 .../org/apache/phoenix/util/SchemaUtil.java     |    6 +
 .../org/apache/phoenix/util/ServerUtil.java     |   95 +-
 .../java/org/apache/phoenix/util/SizedUtil.java |    2 +-
 .../apache/phoenix/util/TransactionUtil.java    |   93 +-
 .../org/apache/phoenix/util/UpgradeUtil.java    |   47 +-
 .../wal/ReadWriteKeyValuesWithCodecTest.java    |    6 +-
 .../apache/phoenix/cache/TenantCacheTest.java   |   14 +-
 .../phoenix/compile/QueryCompilerTest.java      |  815 ++++++++-
 .../phoenix/compile/QueryOptimizerTest.java     |   41 +
 .../phoenix/compile/WhereOptimizerTest.java     |  235 +++
 .../phoenix/execute/CorrelatePlanTest.java      |    4 +-
 .../execute/LiteralResultIteratorPlanTest.java  |    4 +-
 .../expression/ArrayRemoveFunctionTest.java     |  285 +++
 .../function/CollationKeyFunctionTest.java      |   14 +-
 .../expression/function/LowerFunctionTest.java  |   93 +
 .../expression/function/UpperFunctionTest.java  |   92 +
 .../index/covered/CoveredColumnIndexCodec.java  |    8 +-
 .../covered/CoveredIndexCodecForTesting.java    |    5 +-
 .../index/covered/LocalTableStateTest.java      |   38 +-
 .../index/covered/NonTxIndexBuilderTest.java    |    3 +
 .../covered/TestCoveredColumnIndexCodec.java    |   12 +-
 .../hbase/index/write/TestIndexWriter.java      |   25 +-
 .../index/write/TestParalleIndexWriter.java     |   12 +-
 .../write/TestParalleWriterIndexCommitter.java  |   11 +-
 .../index/write/TestWALRecoveryCaching.java     |    7 +-
 .../recovery/TestPerRegionIndexWriteCache.java  |    4 +-
 .../phoenix/memory/MemoryManagerTest.java       |   15 +
 .../java/org/apache/phoenix/query/BaseTest.java |   26 +-
 .../query/ConnectionQueryServicesImplTest.java  |   11 +-
 .../apache/phoenix/query/KeyRangeClipTest.java  |  155 ++
 .../query/ParallelIteratorsSplitTest.java       |    8 +-
 .../phoenix/query/QueryServicesTestImpl.java    |   15 +-
 .../phoenix/schema/PMetaDataImplTest.java       |   21 +-
 .../phoenix/schema/SystemSplitPolicyTest.java   |   97 +
 .../java/org/apache/phoenix/util/TestUtil.java  |   54 +-
 phoenix-flume/pom.xml                           |    8 +-
 phoenix-hive/pom.xml                            |    2 +-
 .../apache/phoenix/hive/HivePhoenixStoreIT.java |   27 +
 .../PhoenixStorageHandlerConstants.java         |    8 +-
 .../hive/mapreduce/PhoenixInputFormat.java      |    3 +-
 .../phoenix/hive/query/PhoenixQueryBuilder.java |    4 +-
 phoenix-kafka/pom.xml                           |   12 +-
 phoenix-load-balancer/pom.xml                   |    2 +-
 phoenix-pherf/pom.xml                           |    6 +-
 .../phoenix/pherf/configuration/Column.java     |   23 +-
 .../pherf/configuration/DataTypeMapping.java    |    6 +-
 .../phoenix/pherf/configuration/Query.java      |   25 +
 .../phoenix/pherf/configuration/Scenario.java   |    3 +-
 .../pherf/configuration/XMLConfigParser.java    |    2 +-
 .../phoenix/pherf/result/QueryResult.java       |   10 +-
 .../phoenix/pherf/result/ResultManager.java     |   18 +-
 .../apache/phoenix/pherf/result/ResultUtil.java |    9 +-
 .../pherf/result/impl/XMLResultHandler.java     |   18 +-
 .../phoenix/pherf/rules/RulesApplier.java       |  142 +-
 .../phoenix/pherf/schema/SchemaReader.java      |    2 +-
 .../apache/phoenix/pherf/util/PhoenixUtil.java  |   44 +-
 .../pherf/workload/MultiThreadedRunner.java     |   35 +-
 .../phoenix/pherf/workload/QueryExecutor.java   |   36 +-
 .../phoenix/pherf/workload/WriteWorkload.java   |   54 +-
 .../scenario/prod_test_unsalted_scenario.xml    |   14 +-
 .../org/apache/phoenix/pherf/ColumnTest.java    |    3 +
 .../phoenix/pherf/ConfigurationParserTest.java  |    4 +-
 .../apache/phoenix/pherf/RuleGeneratorTest.java |  107 +-
 .../test/resources/datamodel/test_schema.sql    |    4 +
 .../test/resources/scenario/test_scenario.xml   |   48 +-
 phoenix-pig/pom.xml                             |    6 +-
 phoenix-protocol/src/main/MetaDataService.proto |    1 +
 phoenix-protocol/src/main/PTable.proto          |    1 +
 .../src/main/ServerCachingService.proto         |    1 +
 phoenix-queryserver-client/pom.xml              |    6 +-
 phoenix-queryserver/pom.xml                     |    6 +-
 .../HttpParamImpersonationQueryServerIT.java    |   16 +-
 .../phoenix/end2end/SecureQueryServerIT.java    |   15 +-
 .../phoenix/queryserver/server/QueryServer.java |   21 +-
 phoenix-server/pom.xml                          |    2 +-
 phoenix-spark/pom.xml                           |    2 +-
 .../phoenix/spark/DataFrameFunctions.scala      |    6 +-
 .../phoenix/spark/ProductRDDFunctions.scala     |    6 +-
 phoenix-tracing-webapp/pom.xml                  |    2 +-
 pom.xml                                         |   35 +-
 python/.gitignore                               |    8 +
 python/.gitlab-ci.yml                           |  149 ++
 python/NEWS.rst                                 |   44 +
 python/README.rst                               |  136 ++
 python/RELEASING.rst                            |   12 +
 python/ci/build-env/Dockerfile                  |    7 +
 python/ci/phoenix/Dockerfile                    |   33 +
 python/ci/phoenix/docker-entrypoint.sh          |   24 +
 python/ci/phoenix/hbase-site.xml                |   12 +
 python/doc/Makefile                             |  192 ++
 python/doc/api.rst                              |   30 +
 python/doc/conf.py                              |  287 +++
 python/doc/index.rst                            |   27 +
 python/doc/versions.rst                         |    3 +
 python/docker-compose.yml                       |   21 +
 python/examples/basic.py                        |   27 +
 python/examples/shell.py                        |   33 +
 python/gen-protobuf.sh                          |   38 +
 python/phoenixdb/__init__.py                    |   68 +
 python/phoenixdb/avatica/__init__.py            |   16 +
 python/phoenixdb/avatica/client.py              |  510 ++++++
 python/phoenixdb/avatica/proto/__init__.py      |    0
 python/phoenixdb/avatica/proto/common_pb2.py    | 1667 ++++++++++++++++++
 python/phoenixdb/avatica/proto/requests_pb2.py  | 1206 +++++++++++++
 python/phoenixdb/avatica/proto/responses_pb2.py |  917 ++++++++++
 python/phoenixdb/connection.py                  |  187 ++
 python/phoenixdb/cursor.py                      |  347 ++++
 python/phoenixdb/errors.py                      |   93 +
 python/phoenixdb/tests/__init__.py              |   44 +
 python/phoenixdb/tests/dbapi20.py               |  857 +++++++++
 python/phoenixdb/tests/test_avatica.py          |   25 +
 python/phoenixdb/tests/test_connection.py       |   42 +
 python/phoenixdb/tests/test_db.py               |   99 ++
 python/phoenixdb/tests/test_dbapi20.py          |  122 ++
 python/phoenixdb/tests/test_errors.py           |   60 +
 python/phoenixdb/tests/test_types.py            |  327 ++++
 python/phoenixdb/types.py                       |  202 +++
 python/requirements.txt                         |   20 +
 python/setup.cfg                                |   34 +
 python/setup.py                                 |   64 +
 python/tox.ini                                  |   24 +
 394 files changed, 23299 insertions(+), 5687 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/f7d87ce2/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------


[30/50] [abbrv] phoenix git commit: PHOENIX-4601 Perform server-side retries if client version < 4.14

Posted by td...@apache.org.
PHOENIX-4601 Perform server-side retries if client version < 4.14


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/78636a3c
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/78636a3c
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/78636a3c

Branch: refs/heads/system-catalog
Commit: 78636a3c25a775d4da2cf52c7cdc7c51ef82bb64
Parents: 5ff900e
Author: Vincent Poon <vi...@apache.org>
Authored: Fri Apr 20 14:22:10 2018 -0700
Committer: Vincent Poon <vi...@apache.org>
Committed: Fri Apr 20 14:27:29 2018 -0700

----------------------------------------------------------------------
 .../hbase/index/write/IndexWriterUtils.java     | 36 ++++++++++++++++----
 .../write/ParallelWriterIndexCommitter.java     | 12 +++++--
 .../TrackingParallelWriterIndexCommitter.java   | 13 ++++---
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |  1 +
 4 files changed, 48 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/78636a3c/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
index 76d6800..0d3004f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
@@ -21,6 +21,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.phoenix.hbase.index.table.HTableFactory;
 import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
 import org.apache.phoenix.util.PropertiesUtil;
@@ -58,15 +59,19 @@ public class IndexWriterUtils {
    public static final String INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY = "phoenix.index.writes.threads.max";
    public static final String HTABLE_KEEP_ALIVE_KEY = "hbase.htable.threads.keepalivetime";
 
+   @Deprecated
    public static final String INDEX_WRITER_RPC_RETRIES_NUMBER = "phoenix.index.writes.rpc.retries.number";
-    /**
-     * Retry server-server index write rpc only once, and let the client retry the data write
-     * instead to avoid typing up the handler
-     */
-   // note in HBase 2+, numTries = numRetries + 1
-   // in prior versions, numTries = numRetries
-   public static final int DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER = 1;
+   /**
+    * Based on the logic in HBase's AsyncProcess, a default of 11 retries with a pause of 100ms
+    * approximates 48 sec total retry time (factoring in backoffs).  The total time should be less
+    * than HBase's rpc timeout (default of 60 sec) or else the client will retry before receiving
+    * the response
+    */
+   @Deprecated
+   public static final int DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER = 11;
+   @Deprecated
    public static final String INDEX_WRITER_RPC_PAUSE = "phoenix.index.writes.rpc.pause";
+   @Deprecated
    public static final int DEFAULT_INDEX_WRITER_RPC_PAUSE = 100;
 
   private IndexWriterUtils() {
@@ -76,12 +81,29 @@ public class IndexWriterUtils {
     public static HTableFactory getDefaultDelegateHTableFactory(CoprocessorEnvironment env) {
         // create a simple delegate factory, setup the way we need
         Configuration conf = PropertiesUtil.cloneConfig(env.getConfiguration());
+        setHTableThreads(conf);
+        return ServerUtil.getDelegateHTableFactory(env, conf);
+    }
+
+    private static void setHTableThreads(Configuration conf) {
         // set the number of threads allowed per table.
         int htableThreads =
                 conf.getInt(IndexWriterUtils.INDEX_WRITER_PER_TABLE_THREADS_CONF_KEY,
                     IndexWriterUtils.DEFAULT_NUM_PER_TABLE_THREADS);
         LOG.trace("Creating HTableFactory with " + htableThreads + " threads for each HTable.");
         IndexManagementUtil.setIfNotSet(conf, HTABLE_THREAD_KEY, htableThreads);
+    }
+
+    /**
+     * Retry server-server index write rpc only once, and let the client retry the data write
+     * instead to avoid tying up the handler
+     */
+    public static HTableFactory getNoRetriesHTableFactory(CoprocessorEnvironment env) {
+        Configuration conf = PropertiesUtil.cloneConfig(env.getConfiguration());
+        setHTableThreads(conf);
+        // note in HBase 2+, numTries = numRetries + 1
+        // in prior versions, numTries = numRetries
+        conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
         return ServerUtil.getDelegateHTableFactory(env, conf);
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/78636a3c/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
index aba2678..e06efcc 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
@@ -36,6 +36,7 @@ import org.apache.phoenix.hbase.index.table.HTableFactory;
 import org.apache.phoenix.hbase.index.table.HTableInterfaceReference;
 import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
 import org.apache.phoenix.index.PhoenixIndexFailurePolicy;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.util.IndexUtil;
 
 import com.google.common.collect.Multimap;
@@ -57,7 +58,8 @@ public class ParallelWriterIndexCommitter implements IndexCommitter {
     public static final String INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY = "index.writer.threads.keepalivetime";
     private static final Log LOG = LogFactory.getLog(ParallelWriterIndexCommitter.class);
 
-    private HTableFactory factory;
+    private HTableFactory retryingFactory;
+    private HTableFactory noRetriesfactory;
     private Stoppable stopped;
     private QuickFailingTaskRunner pool;
     private KeyValueBuilder kvBuilder;
@@ -88,7 +90,8 @@ public class ParallelWriterIndexCommitter implements IndexCommitter {
      * Exposed for TESTING
      */
     void setup(HTableFactory factory, ExecutorService pool, Abortable abortable, Stoppable stop, RegionCoprocessorEnvironment env) {
-        this.factory = factory;
+        this.retryingFactory = factory;
+        this.noRetriesfactory = IndexWriterUtils.getNoRetriesHTableFactory(env);
         this.pool = new QuickFailingTaskRunner(pool);
         this.stopped = stop;
     }
@@ -162,6 +165,8 @@ public class ParallelWriterIndexCommitter implements IndexCommitter {
                                 }
                             }
                         }
+                     // if the client can retry index writes, then we don't need to retry here
+                        HTableFactory factory = clientVersion < PhoenixDatabaseMetaData.MIN_CLIENT_RETRY_INDEX_WRITES ? retryingFactory : noRetriesfactory;
                         table = factory.getTable(tableReference.get());
                         throwFailureIfDone();
                         table.batch(mutations);
@@ -226,7 +231,8 @@ public class ParallelWriterIndexCommitter implements IndexCommitter {
     public void stop(String why) {
         LOG.info("Shutting down " + this.getClass().getSimpleName() + " because " + why);
         this.pool.stop(why);
-        this.factory.shutdown();
+        this.retryingFactory.shutdown();
+        this.noRetriesfactory.shutdown();
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/phoenix/blob/78636a3c/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
index 4dbad63..4ba1155 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
@@ -41,6 +41,7 @@ import org.apache.phoenix.hbase.index.table.HTableFactory;
 import org.apache.phoenix.hbase.index.table.HTableInterfaceReference;
 import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
 import org.apache.phoenix.index.PhoenixIndexFailurePolicy;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.util.IndexUtil;
 
 import com.google.common.collect.Multimap;
@@ -74,7 +75,8 @@ public class TrackingParallelWriterIndexCommitter implements IndexCommitter {
     private static final String INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY = "index.writer.threads.keepalivetime";
 
     private TaskRunner pool;
-    private HTableFactory factory;
+    private HTableFactory retryingFactory;
+    private HTableFactory noRetriesFactory;
     private CapturingAbortable abortable;
     private Stoppable stopped;
     private RegionCoprocessorEnvironment env;
@@ -108,7 +110,8 @@ public class TrackingParallelWriterIndexCommitter implements IndexCommitter {
     void setup(HTableFactory factory, ExecutorService pool, Abortable abortable, Stoppable stop,
             RegionCoprocessorEnvironment env) {
         this.pool = new WaitForCompletionTaskRunner(pool);
-        this.factory = factory;
+        this.retryingFactory = factory;
+        this.noRetriesFactory = IndexWriterUtils.getNoRetriesHTableFactory(env);
         this.abortable = new CapturingAbortable(abortable);
         this.stopped = stop;
         this.env = env;
@@ -175,7 +178,8 @@ public class TrackingParallelWriterIndexCommitter implements IndexCommitter {
                         if (LOG.isTraceEnabled()) {
                             LOG.trace("Writing index update:" + mutations + " to table: " + tableReference);
                         }
-
+                        // if the client can retry index writes, then we don't need to retry here
+                        HTableFactory factory = clientVersion < PhoenixDatabaseMetaData.MIN_CLIENT_RETRY_INDEX_WRITES ? retryingFactory : noRetriesFactory;
                         table = factory.getTable(tableReference.get());
                         throwFailureIfDone();
                         table.batch(mutations);
@@ -238,7 +242,8 @@ public class TrackingParallelWriterIndexCommitter implements IndexCommitter {
     public void stop(String why) {
         LOG.info("Shutting down " + this.getClass().getSimpleName());
         this.pool.stop(why);
-        this.factory.shutdown();
+        this.retryingFactory.shutdown();
+        this.noRetriesFactory.shutdown();
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/phoenix/blob/78636a3c/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index add0628..d56628a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -327,6 +327,7 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
     public static final int MIN_NAMESPACE_MAPPED_PHOENIX_VERSION = VersionUtil.encodeVersion("4", "8", "0");
     public static final int MIN_PENDING_ACTIVE_INDEX = VersionUtil.encodeVersion("4", "12", "0");
     public static final int MIN_PENDING_DISABLE_INDEX = VersionUtil.encodeVersion("4", "14", "0");
+    public static final int MIN_CLIENT_RETRY_INDEX_WRITES = VersionUtil.encodeVersion("4", "14", "0");
     public static final int MIN_TX_CLIENT_SIDE_MAINTENANCE = VersionUtil.encodeVersion("4", "14", "0");
     
     // Version below which we should turn off essential column family.


[47/50] [abbrv] phoenix git commit: PHOENIX-4709 Alter split policy in upgrade path for system tables

Posted by td...@apache.org.
PHOENIX-4709 Alter split policy in upgrade path for system tables


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/fc194c56
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/fc194c56
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/fc194c56

Branch: refs/heads/system-catalog
Commit: fc194c568ee5a4a7ba0e92b64ae24f7cf4b224e5
Parents: a18fd1e
Author: James Taylor <jt...@salesforce.com>
Authored: Thu Apr 26 13:38:05 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Apr 27 11:41:56 2018 -0700

----------------------------------------------------------------------
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |   1 +
 .../query/ConnectionQueryServicesImpl.java      |  17 +++
 .../apache/phoenix/query/QueryConstants.java    | 109 ++++++++++++++++++-
 3 files changed, 124 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc194c56/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index d56628a..320c6e7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -217,6 +217,7 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
 
     public static final String TYPE_SEQUENCE = "SEQUENCE";
     public static final String SYSTEM_FUNCTION_TABLE = "FUNCTION";
+    public static final String SYSTEM_FUNCTION = SYSTEM_CATALOG_SCHEMA + QueryConstants.NAME_SEPARATOR + "\"FUNCTION\"";
     public static final String SYSTEM_FUNCTION_NAME = SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_FUNCTION_TABLE);
     public static final byte[] SYSTEM_FUNCTION_NAME_BYTES = Bytes.toBytes(SYSTEM_FUNCTION_NAME);
     public static final TableName SYSTEM_FUNCTION_HBASE_TABLE_NAME = TableName.valueOf(SYSTEM_FUNCTION_NAME);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc194c56/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index d841f0b..1d3feed 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -214,6 +214,8 @@ import org.apache.phoenix.schema.Sequence;
 import org.apache.phoenix.schema.SequenceAllocation;
 import org.apache.phoenix.schema.SequenceKey;
 import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.SystemFunctionSplitPolicy;
+import org.apache.phoenix.schema.SystemStatsSplitPolicy;
 import org.apache.phoenix.schema.TableAlreadyExistsException;
 import org.apache.phoenix.schema.TableNotFoundException;
 import org.apache.phoenix.schema.TableProperty;
@@ -2949,6 +2951,21 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0,
               PhoenixDatabaseMetaData.TRANSACTION_PROVIDER + " "
                 + PTinyint.INSTANCE.getSqlTypeName());
+            metaConnection.createStatement().executeUpdate("ALTER TABLE " + 
+                    PhoenixDatabaseMetaData.SYSTEM_CATALOG + " SET " + 
+                    HConstants.VERSIONS + "= " + props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS) + ",\n" +
+                    HColumnDescriptor.KEEP_DELETED_CELLS + "=" + props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS)
+                    );
+            metaConnection.createStatement().executeUpdate("ALTER TABLE " + 
+                PhoenixDatabaseMetaData.SYSTEM_FUNCTION + " SET " + 
+                    HTableDescriptor.SPLIT_POLICY + "='" + SystemFunctionSplitPolicy.class.getName() + "',\n" +
+                    HConstants.VERSIONS + "= " + props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS) + ",\n" +
+                    HColumnDescriptor.KEEP_DELETED_CELLS + "=" + props.getBoolean(DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_KEEP_DELETED_CELLS)
+                    );
+            metaConnection.createStatement().executeUpdate("ALTER TABLE " + 
+                    PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " SET " + 
+                    HTableDescriptor.SPLIT_POLICY + "='" + SystemStatsSplitPolicy.class.getName() +"'"
+                    );
         }
         return metaConnection;
     }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc194c56/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index 4285334..2fe7b14 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -18,7 +18,112 @@
 package org.apache.phoenix.query;
 
 
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.*;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.AUTO_PARTITION_SEQ;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BASE_COLUMN_COUNT;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BIND_PARAMETERS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BUFFER_LENGTH;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CACHE_SIZE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CHAR_OCTET_LENGTH;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLASS_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLIENT_IP;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CURRENT_VALUE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CYCLE_FLAG;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ENCODING_SCHEME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXCEPTION_TRACE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXPLAIN_PLAN;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GLOBAL_SCAN_DETAILS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POST_KEY;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INCREMENT_BY;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_ARRAY;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_AUTOINCREMENT;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_CONSTANT;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_NAMESPACE_MAPPED;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_NULLABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_ROW_TIMESTAMP;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_VIEW_REFERENCED;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.JAR_PATH;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.KEY_SEQ;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LAST_STATS_UPDATE_TIME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LIMIT_REACHED_FLAG;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MAX_VALUE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MIN_VALUE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MULTI_TENANT;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NO_OF_RESULTS_ITERATED;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NULLABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_ARGS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_PREC_RADIX;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PHYSICAL_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PK_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_ID;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_STATUS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.REF_GENERATION;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.REMARKS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.RETURN_TYPE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SALT_BUCKETS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCAN_METRICS_JSON;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_CATALOG;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_SCHEMA;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_TABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SELF_REFERENCING_COL_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SEQUENCE_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SEQUENCE_SCHEMA;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SOURCE_DATA_TYPE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATA_TYPE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATETIME_SUB;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_TIME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_WITH;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STORE_NULLS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_TABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TOTAL_EXECUTION_TIME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTION_PROVIDER;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USER;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
 
 import java.math.BigDecimal;
 
@@ -326,8 +431,6 @@ public interface QueryConstants {
             BIND_PARAMETERS + " VARCHAR, \n" +
             SCAN_METRICS_JSON + " VARCHAR, \n" +
             " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (QUERY_ID))\n" +
-            HConstants.VERSIONS + "= " + MetaDataProtocol.DEFAULT_LOG_VERSIONS + ",\n" +
-            HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n"+
             PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE+ ",\n" +
             HColumnDescriptor.TTL + "=" + MetaDataProtocol.DEFAULT_LOG_TTL+",\n"+
             TableProperty.COLUMN_ENCODED_BYTES.toString()+" = 0";


[16/50] [abbrv] phoenix git commit: PHOENIX-4496 Fix RowValueConstructorIT and IndexMetadataIT

Posted by td...@apache.org.
PHOENIX-4496 Fix RowValueConstructorIT and IndexMetadataIT


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/70c9c6bb
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/70c9c6bb
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/70c9c6bb

Branch: refs/heads/system-catalog
Commit: 70c9c6bb2d2073bb40640a0a794032b166c4b63b
Parents: a0b0c1e
Author: Ankit Singhal <an...@gmail.com>
Authored: Fri Apr 13 12:09:30 2018 +0530
Committer: Ankit Singhal <an...@gmail.com>
Committed: Fri Apr 13 12:09:30 2018 +0530

----------------------------------------------------------------------
 .../phoenix/hbase/index/scanner/ScannerBuilder.java       | 10 ++++++++++
 1 file changed, 10 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/70c9c6bb/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
index ad09c0c..4c42fe4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/scanner/ScannerBuilder.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.FamilyFilter;
 import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
 import org.apache.hadoop.hbase.filter.FilterList;
 import org.apache.hadoop.hbase.filter.QualifierFilter;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -98,6 +99,15 @@ public class ScannerBuilder {
       }
       columnFilters.addFilter(columnFilter);
     }
+    
+    if(columns.isEmpty()){
+        columnFilters.addFilter(new FilterBase(){
+            @Override
+            public boolean filterAllRemaining() throws IOException {
+                return true;
+            }
+        });
+    }
     return columnFilters;
   }
 


[48/50] [abbrv] phoenix git commit: PHOENIX-4716 ParameterizedTransactionIT is failing in 0.98 branch

Posted by td...@apache.org.
PHOENIX-4716 ParameterizedTransactionIT is failing in 0.98 branch


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d10151e3
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d10151e3
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d10151e3

Branch: refs/heads/system-catalog
Commit: d10151e33615cacd01e634fef896e8644fced890
Parents: fc194c5
Author: James Taylor <jt...@salesforce.com>
Authored: Fri Apr 27 18:25:56 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Apr 27 18:37:16 2018 -0700

----------------------------------------------------------------------
 .../phoenix/tx/ParameterizedTransactionIT.java  | 37 +++++++++++---------
 1 file changed, 20 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/d10151e3/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
index ce01e2b..580f055 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
@@ -71,12 +71,15 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
     private final String tableDDLOptions;
 
     public ParameterizedTransactionIT(boolean mutable, boolean columnEncoded) {
-        StringBuilder optionBuilder = new StringBuilder("TRANSACTIONAL=true");
+        StringBuilder optionBuilder = new StringBuilder();
         if (!columnEncoded) {
-            optionBuilder.append(",COLUMN_ENCODED_BYTES=0");
+            optionBuilder.append("COLUMN_ENCODED_BYTES=0");
         }
         if (!mutable) {
-            optionBuilder.append(",IMMUTABLE_ROWS=true");
+            if (optionBuilder.length() > 0) {
+                optionBuilder.append(",");
+            }
+            optionBuilder.append("IMMUTABLE_ROWS=true");
             if (!columnEncoded) {
                 optionBuilder.append(",IMMUTABLE_STORAGE_SCHEME="+PTableImpl.ImmutableStorageScheme.ONE_CELL_PER_COLUMN);
             }
@@ -97,7 +100,7 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + transTableName;
         String selectSql = "SELECT * FROM "+ fullTableName;
         try (Connection conn = DriverManager.getConnection(getUrl())) {
-            conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions);
+            conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions + (tableDDLOptions.length() > 0 ? "," : "") + "TRANSACTIONAL=true");
             conn.setAutoCommit(false);
             ResultSet rs = conn.createStatement().executeQuery(selectSql);
             assertFalse(rs.next());
@@ -132,7 +135,7 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + transTableName;
         String selectSql = "SELECT * FROM "+fullTableName;
         try (Connection conn = DriverManager.getConnection(getUrl())) {
-            conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions);
+            conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions + (tableDDLOptions.length() > 0 ? "," : "") + "TRANSACTIONAL=true");
             conn.setAutoCommit(false);
             ResultSet rs = conn.createStatement().executeQuery(selectSql);
             assertFalse(rs.next());
@@ -165,7 +168,7 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         String transTableName = generateUniqueName();
         String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + transTableName;
         try (Connection conn = DriverManager.getConnection(getUrl())) {
-            conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions);
+            conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions + (tableDDLOptions.length() > 0 ? "," : "") + "TRANSACTIONAL=true");
             conn.setAutoCommit(true);
             // verify no rows returned
             ResultSet rs = conn.createStatement().executeQuery("SELECT * FROM " + fullTableName);
@@ -178,7 +181,7 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         String transTableName = generateUniqueName();
         String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + transTableName;
         try (Connection conn = DriverManager.getConnection(getUrl())) {
-            conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions);
+            conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions + (tableDDLOptions.length() > 0 ? "," : "") + "TRANSACTIONAL=true");
             conn.setAutoCommit(true);
             // verify no rows returned
             ResultSet rs = conn.createStatement().executeQuery("SELECT * FROM " + fullTableName + " x JOIN " + fullTableName + " y ON (x.long_pk = y.int_pk)");
@@ -191,8 +194,8 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         String t1 = generateUniqueName();
         String t2 = generateUniqueName();
         try (Connection conn = DriverManager.getConnection(getUrl())) {
-            conn.createStatement().execute("create table " + t1 + " (varchar_pk VARCHAR NOT NULL primary key, a.varchar_col1 VARCHAR, b.varchar_col2 VARCHAR)" + tableDDLOptions);
-            conn.createStatement().execute("create table " + t2 + " (varchar_pk VARCHAR NOT NULL primary key, a.varchar_col1 VARCHAR, b.varchar_col1 VARCHAR)" + tableDDLOptions);
+            conn.createStatement().execute("create table " + t1 + " (varchar_pk VARCHAR NOT NULL primary key, a.varchar_col1 VARCHAR, b.varchar_col2 VARCHAR)" + tableDDLOptions + (tableDDLOptions.length() > 0 ? "," : "") + "TRANSACTIONAL=true");
+            conn.createStatement().execute("create table " + t2 + " (varchar_pk VARCHAR NOT NULL primary key, a.varchar_col1 VARCHAR, b.varchar_col1 VARCHAR)" + tableDDLOptions + (tableDDLOptions.length() > 0 ? "," : "") + "TRANSACTIONAL=true");
             // verify no rows returned
             ResultSet rs = conn.createStatement().executeQuery("SELECT * FROM " + t1 + " x JOIN " + t1 + " y ON (x.varchar_pk = y.a.varchar_col1)");
             assertFalse(rs.next());
@@ -242,7 +245,7 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         String transTableName = generateUniqueName();
         String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + transTableName;
         Connection conn = DriverManager.getConnection(getUrl());
-        conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions);
+        conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions + (tableDDLOptions.length() > 0 ? "," : "") + "TRANSACTIONAL=true");
         testRowConflicts(fullTableName);
     }
     
@@ -251,7 +254,7 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         String transTableName = generateUniqueName();
         String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + transTableName;
         Connection conn = DriverManager.getConnection(getUrl());
-        conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions);
+        conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions + (tableDDLOptions.length() > 0 ? "," : "") + "TRANSACTIONAL=true");
         conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET IMMUTABLE_ROWS=true");
         testRowConflicts(fullTableName);
     }
@@ -369,7 +372,7 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         Connection conn = DriverManager.getConnection(getUrl(), props);
         String t1 = generateUniqueName();
         String t2 = generateUniqueName();
-        String ddl = "CREATE TABLE " + t1 + " (k varchar primary key) " + tableDDLOptions;
+        String ddl = "CREATE TABLE " + t1 + " (k varchar primary key)" + tableDDLOptions + (tableDDLOptions.length() > 0 ? "," : "") + "TRANSACTIONAL=true";
         conn.createStatement().execute(ddl);
         PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
         PTable table = pconn.getTable(new PTableKey(null, t1));
@@ -418,7 +421,7 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + transTableName;
         String selectSql = "SELECT current_date() FROM "+fullTableName;
         try (Connection conn = DriverManager.getConnection(getUrl())) {
-            conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions);
+            conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions + (tableDDLOptions.length() > 0 ? "," : "") + "TRANSACTIONAL=true");
             conn.setAutoCommit(false);
             ResultSet rs = conn.createStatement().executeQuery(selectSql);
             assertFalse(rs.next());
@@ -458,9 +461,9 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         String fullTableName2 = generateUniqueName();
         String sequenceName = "S_" + generateUniqueName();
         conn.createStatement().execute("CREATE SEQUENCE " + sequenceName);
-        conn.createStatement().execute("CREATE TABLE " + fullTableName1 + " (pk INTEGER PRIMARY KEY, val INTEGER) SALT_BUCKETS=4"
-                + (!tableDDLOptions.isEmpty()? "," : "") + tableDDLOptions);
-        conn.createStatement().execute("CREATE TABLE " + fullTableName2 + " (pk INTEGER PRIMARY KEY, val INTEGER)" + tableDDLOptions);
+        conn.createStatement().execute("CREATE TABLE " + fullTableName1 + " (pk INTEGER PRIMARY KEY, val INTEGER) SALT_BUCKETS=4, TRANSACTIONAL=true"
+                + (tableDDLOptions.length() > 0 ? "," : "")  + tableDDLOptions);
+        conn.createStatement().execute("CREATE TABLE " + fullTableName2 + " (pk INTEGER PRIMARY KEY, val INTEGER)" );
 
         for (int i = 0; i < 100; i++) {
             conn.createStatement().execute("UPSERT INTO " + fullTableName1 + " VALUES (NEXT VALUE FOR " + sequenceName + ", " + (i%10) + ")");
@@ -478,7 +481,7 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         try (Connection conn = DriverManager.getConnection(getUrl())) {
             String transactTableName = generateUniqueName();
             Statement stmt = conn.createStatement();
-            stmt.execute("CREATE TABLE " + transactTableName + " (k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) " + tableDDLOptions);
+            stmt.execute("CREATE TABLE " + transactTableName + " (k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) " + tableDDLOptions + (tableDDLOptions.length() > 0 ? "," : "") + "TRANSACTIONAL=true");
 
             
             try (Connection conn1 = DriverManager.getConnection(getUrl()); Connection conn2 = DriverManager.getConnection(getUrl())) {


[06/50] [abbrv] phoenix git commit: PHOENIX-2715 Query Log (addendum) Fix old com.lmax.disruptor dependency

Posted by td...@apache.org.
PHOENIX-2715 Query Log (addendum)
Fix old com.lmax.disruptor dependency


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d48518bc
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d48518bc
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d48518bc

Branch: refs/heads/system-catalog
Commit: d48518bc2ddea3f4c6cd9ef5a6c2504b65255da2
Parents: 2e6dc81
Author: Ankit Singhal <an...@gmail.com>
Authored: Wed Apr 11 17:36:03 2018 +0530
Committer: Ankit Singhal <an...@gmail.com>
Committed: Wed Apr 11 17:36:03 2018 +0530

----------------------------------------------------------------------
 phoenix-core/pom.xml  | 7 -------
 phoenix-flume/pom.xml | 6 ------
 phoenix-kafka/pom.xml | 6 ------
 3 files changed, 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/d48518bc/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index 76ca9c9..9e9a15b 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -345,12 +345,6 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>com.lmax</groupId>
-      <artifactId>disruptor</artifactId>
-      <version>3.2.0</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
       <groupId>org.apache.htrace</groupId>
       <artifactId>htrace-core</artifactId>
     </dependency>
@@ -483,7 +477,6 @@
        <dependency>
         <groupId>com.lmax</groupId>
         <artifactId>disruptor</artifactId>
-        <version>${disruptor.version}</version>
       </dependency>
   </dependencies>
 </project>

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d48518bc/phoenix-flume/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-flume/pom.xml b/phoenix-flume/pom.xml
index 357bc2d..3780d8d 100644
--- a/phoenix-flume/pom.xml
+++ b/phoenix-flume/pom.xml
@@ -69,12 +69,6 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>com.lmax</groupId>
-      <artifactId>disruptor</artifactId>
-      <version>3.2.0</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
       <groupId>org.apache.htrace</groupId>
       <artifactId>htrace-core</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d48518bc/phoenix-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-kafka/pom.xml b/phoenix-kafka/pom.xml
index bdcaef4..09547c5 100644
--- a/phoenix-kafka/pom.xml
+++ b/phoenix-kafka/pom.xml
@@ -143,12 +143,6 @@
 			<scope>test</scope>
 		</dependency>
 		<dependency>
-			<groupId>com.lmax</groupId>
-			<artifactId>disruptor</artifactId>
-			<version>3.2.0</version>
-			<scope>test</scope>
-		</dependency>
-		<dependency>
 			<groupId>org.apache.htrace</groupId>
 			<artifactId>htrace-core</artifactId>
 		</dependency>


[26/50] [abbrv] phoenix git commit: PHOENIX-4600 Add retry logic for partial index rebuilder writes

Posted by td...@apache.org.
PHOENIX-4600 Add retry logic for partial index rebuilder writes


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b768900d
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b768900d
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b768900d

Branch: refs/heads/system-catalog
Commit: b768900d9f9699fad612acc643866e3e420e7ce0
Parents: fc6cf43
Author: Vincent Poon <vi...@apache.org>
Authored: Thu Apr 19 10:30:14 2018 -0700
Committer: Vincent Poon <vi...@apache.org>
Committed: Thu Apr 19 12:03:34 2018 -0700

----------------------------------------------------------------------
 .../end2end/index/MutableIndexRebuilderIT.java  | 143 +++++++++++++++++++
 .../UngroupedAggregateRegionObserver.java       |  32 +++--
 2 files changed, 160 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/b768900d/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexRebuilderIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexRebuilderIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexRebuilderIT.java
new file mode 100644
index 0000000..8420f16
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexRebuilderIT.java
@@ -0,0 +1,143 @@
+package org.apache.phoenix.end2end.index;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver;
+import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
+import org.apache.phoenix.coprocessor.MetaDataRegionObserver;
+import org.apache.phoenix.coprocessor.MetaDataRegionObserver.BuildIndexScheduleTask;
+import org.apache.phoenix.end2end.BaseUniqueNamesOwnClusterIT;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.PIndexState;
+import org.apache.phoenix.util.EnvironmentEdgeManager;
+import org.apache.phoenix.util.IndexUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.RunUntilFailure;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.google.common.collect.Maps;
+
+@RunWith(RunUntilFailure.class)
+public class MutableIndexRebuilderIT extends BaseUniqueNamesOwnClusterIT {
+    private static final int WAIT_AFTER_DISABLED = 0;
+    private static final long REBUILD_PERIOD = 50000;
+    private static final long REBUILD_INTERVAL = 2000;
+    private static RegionCoprocessorEnvironment indexRebuildTaskRegionEnvironment;
+
+    /**
+     * Tests that the index rebuilder retries for exactly the configured # of retries
+     * @throws Exception
+     */
+    @Test
+    public void testRebuildRetriesSuccessful() throws Throwable {
+        int numberOfRetries = 5;
+        Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(10);
+        serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_ATTRIB, Boolean.TRUE.toString());
+        serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_INTERVAL_ATTRIB, Long.toString(REBUILD_INTERVAL));
+        serverProps.put(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD, "50000000");
+        serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_PERIOD, Long.toString(REBUILD_PERIOD)); // batch at 50 seconds
+        serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB, Long.toString(WAIT_AFTER_DISABLED));
+        serverProps.put(HConstants.HBASE_CLIENT_RETRIES_NUMBER, numberOfRetries + "");
+        Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(1);
+        setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator()));
+        indexRebuildTaskRegionEnvironment =
+                (RegionCoprocessorEnvironment) getUtility()
+                .getRSForFirstRegionInTable(
+                    PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME)
+                .getOnlineRegions(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME)
+                .get(0).getCoprocessorHost()
+                .findCoprocessorEnvironment(MetaDataRegionObserver.class.getName());
+        MetaDataRegionObserver.initRebuildIndexConnectionProps(
+            indexRebuildTaskRegionEnvironment.getConfiguration());
+        try (Connection conn = DriverManager.getConnection(getUrl())) {
+            String schemaName = generateUniqueName();
+            String tableName = generateUniqueName();
+            String indexName = generateUniqueName();
+            final String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
+            final String fullIndexName = SchemaUtil.getTableName(schemaName, indexName);
+            conn.createStatement().execute("CREATE TABLE " + fullTableName + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR, v3 VARCHAR) DISABLE_INDEX_ON_WRITE_FAILURE = TRUE");
+            conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + fullTableName + " (v1, v2)");
+            HTableInterface metaTable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES);
+            IndexUtil.updateIndexState(fullIndexName, EnvironmentEdgeManager.currentTimeMillis(), metaTable, PIndexState.DISABLE);
+            conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES('a','a','0')");
+            conn.commit();
+            // Simulate write failure when rebuilder runs
+            TestUtil.addCoprocessor(conn, fullIndexName, WriteFailingRegionObserver.class);
+            waitForIndexState(conn, fullTableName, fullIndexName, PIndexState.INACTIVE);
+            // rebuild writes should retry for exactly the configured number of times
+            ExecutorService executor = Executors.newSingleThreadExecutor();
+            try {
+                Future<Boolean> future = executor.submit(new Callable<Boolean>() {
+                    @Override
+                    public Boolean call() throws Exception {
+                        runIndexRebuilder(fullTableName);
+                        return true;
+                    }});
+                assertTrue(future.get(120, TimeUnit.SECONDS));
+                assertEquals(numberOfRetries, WriteFailingRegionObserver.attempts.get());
+            } finally {
+                executor.shutdownNow();
+            }
+        }
+    }
+
+    public static void waitForIndexState(Connection conn, String fullTableName, String fullIndexName, PIndexState expectedIndexState) throws InterruptedException, SQLException {
+        int nRetries = 2;
+        PIndexState actualIndexState = null;
+        do {
+            runIndexRebuilder(fullTableName);
+            if ((actualIndexState = TestUtil.getIndexState(conn, fullIndexName)) == expectedIndexState) {
+                return;
+            }
+            Thread.sleep(1000);
+        } while (--nRetries > 0);
+        fail("Expected index state of " + expectedIndexState + ", but was " + actualIndexState);
+    }
+
+    private static void runIndexRebuilder(String table) throws InterruptedException, SQLException {
+        runIndexRebuilder(Collections.<String>singletonList(table));
+    }
+
+    private static void runIndexRebuilder(List<String> tables) throws InterruptedException, SQLException {
+        BuildIndexScheduleTask task =
+                new MetaDataRegionObserver.BuildIndexScheduleTask(
+                        indexRebuildTaskRegionEnvironment, tables);
+        task.run();
+    }
+
+    public static class WriteFailingRegionObserver extends SimpleRegionObserver {
+        public static volatile AtomicInteger attempts = new AtomicInteger(0);
+        @Override
+        public void postBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c, MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
+            attempts.incrementAndGet();
+            throw new DoNotRetryIOException("Simulating write failure on " + c.getEnvironment().getRegionInfo().getTable().getNameAsString());
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/b768900d/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index d202193..abdcf72 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -243,6 +243,19 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
         indexWriteProps = new ReadOnlyProps(indexWriteConfig.iterator());
     }
 
+    private void commitBatchWithRetries(final Region region, final List<Mutation> localRegionMutations, final long blockingMemstoreSize) throws IOException {
+        try {
+            commitBatch(region, localRegionMutations, blockingMemstoreSize);
+        } catch (IOException e) {
+            handleIndexWriteException(localRegionMutations, e, new MutateCommand() {
+                @Override
+                public void doMutation() throws IOException {
+                    commitBatch(region, localRegionMutations, blockingMemstoreSize);
+                }
+            });
+        }
+    }
+
     private void commitBatch(Region region, List<Mutation> mutations, long blockingMemstoreSize) throws IOException {
       if (mutations.isEmpty()) {
           return;
@@ -251,7 +264,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
         Mutation[] mutationArray = new Mutation[mutations.size()];
       // When memstore size reaches blockingMemstoreSize we are waiting 3 seconds for the
       // flush happen which decrease the memstore size and then writes allowed on the region.
-      for (int i = 0; region.getMemstoreSize() > blockingMemstoreSize && i < 30; i++) {
+      for (int i = 0; blockingMemstoreSize > 0 && region.getMemstoreSize() > blockingMemstoreSize && i < 30; i++) {
           try {
               checkForRegionClosing();
               Thread.sleep(100);
@@ -892,16 +905,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
         setIndexAndTransactionProperties(mutations, indexUUID, indexMaintainersPtr, txState, clientVersionBytes, useIndexProto);
         separateLocalAndRemoteMutations(targetHTable, region, mutations, localRegionMutations, remoteRegionMutations,
             isPKChanging);
-        try {
-            commitBatch(region, localRegionMutations, blockingMemStoreSize);
-        } catch (IOException e) {
-            handleIndexWriteException(localRegionMutations, e, new MutateCommand() {
-                @Override
-                public void doMutation() throws IOException {
-                    commitBatch(region, localRegionMutations, blockingMemStoreSize);
-                }
-            });
-        }
+        commitBatchWithRetries(region, localRegionMutations, blockingMemStoreSize);
         try {
             commitBatchWithHTable(targetHTable, remoteRegionMutations);
         } catch (IOException e) {
@@ -1069,8 +1073,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
                             }
                         }
                         if (ServerUtil.readyToCommit(mutations.size(), mutations.byteSize(), maxBatchSize, maxBatchSizeBytes)) {
-                            region.batchMutate(mutations.toArray(new Mutation[mutations.size()]), HConstants.NO_NONCE,
-                                    HConstants.NO_NONCE);
+                            commitBatchWithRetries(region, mutations, -1);
                             uuidValue = ServerCacheClient.generateId();
                             mutations.clear();
                         }
@@ -1079,8 +1082,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
                     
                 } while (hasMore);
                 if (!mutations.isEmpty()) {
-                    region.batchMutate(mutations.toArray(new Mutation[mutations.size()]), HConstants.NO_NONCE,
-                            HConstants.NO_NONCE);
+                    commitBatchWithRetries(region, mutations, -1);
                 }
             }
         } catch (IOException e) {


[29/50] [abbrv] phoenix git commit: Revert "PHOENIX-4601 Perform server-side retries if client version < 4.14"

Posted by td...@apache.org.
Revert "PHOENIX-4601 Perform server-side retries if client version < 4.14"

This reverts commit d7afec21c500ccd7f5334e9c0958c40fcc95df14.


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/5ff900e5
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/5ff900e5
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/5ff900e5

Branch: refs/heads/system-catalog
Commit: 5ff900e57c5040ea9ee42d0ca48c0cf68cf86b30
Parents: d7afec2
Author: Vincent Poon <vi...@apache.org>
Authored: Fri Apr 20 14:26:27 2018 -0700
Committer: Vincent Poon <vi...@apache.org>
Committed: Fri Apr 20 14:26:27 2018 -0700

----------------------------------------------------------------------
 .../hbase/index/write/IndexWriterUtils.java     | 36 ++++----------------
 .../write/ParallelWriterIndexCommitter.java     | 12 ++-----
 .../TrackingParallelWriterIndexCommitter.java   | 13 +++----
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |  1 -
 4 files changed, 14 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/5ff900e5/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
index 0d3004f..76d6800 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
@@ -21,7 +21,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.phoenix.hbase.index.table.HTableFactory;
 import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
 import org.apache.phoenix.util.PropertiesUtil;
@@ -59,19 +58,15 @@ public class IndexWriterUtils {
    public static final String INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY = "phoenix.index.writes.threads.max";
    public static final String HTABLE_KEEP_ALIVE_KEY = "hbase.htable.threads.keepalivetime";
 
-   @Deprecated
    public static final String INDEX_WRITER_RPC_RETRIES_NUMBER = "phoenix.index.writes.rpc.retries.number";
-   /**
-    * Based on the logic in HBase's AsyncProcess, a default of 11 retries with a pause of 100ms
-    * approximates 48 sec total retry time (factoring in backoffs).  The total time should be less
-    * than HBase's rpc timeout (default of 60 sec) or else the client will retry before receiving
-    * the response
-    */
-   @Deprecated
-   public static final int DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER = 11;
-   @Deprecated
+    /**
+     * Retry server-server index write rpc only once, and let the client retry the data write
+     * instead to avoid typing up the handler
+     */
+   // note in HBase 2+, numTries = numRetries + 1
+   // in prior versions, numTries = numRetries
+   public static final int DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER = 1;
    public static final String INDEX_WRITER_RPC_PAUSE = "phoenix.index.writes.rpc.pause";
-   @Deprecated
    public static final int DEFAULT_INDEX_WRITER_RPC_PAUSE = 100;
 
   private IndexWriterUtils() {
@@ -81,29 +76,12 @@ public class IndexWriterUtils {
     public static HTableFactory getDefaultDelegateHTableFactory(CoprocessorEnvironment env) {
         // create a simple delegate factory, setup the way we need
         Configuration conf = PropertiesUtil.cloneConfig(env.getConfiguration());
-        setHTableThreads(conf);
-        return ServerUtil.getDelegateHTableFactory(env, conf);
-    }
-
-    private static void setHTableThreads(Configuration conf) {
         // set the number of threads allowed per table.
         int htableThreads =
                 conf.getInt(IndexWriterUtils.INDEX_WRITER_PER_TABLE_THREADS_CONF_KEY,
                     IndexWriterUtils.DEFAULT_NUM_PER_TABLE_THREADS);
         LOG.trace("Creating HTableFactory with " + htableThreads + " threads for each HTable.");
         IndexManagementUtil.setIfNotSet(conf, HTABLE_THREAD_KEY, htableThreads);
-    }
-
-    /**
-     * Retry server-server index write rpc only once, and let the client retry the data write
-     * instead to avoid tying up the handler
-     */
-    public static HTableFactory getNoRetriesHTableFactory(CoprocessorEnvironment env) {
-        Configuration conf = PropertiesUtil.cloneConfig(env.getConfiguration());
-        setHTableThreads(conf);
-        // note in HBase 2+, numTries = numRetries + 1
-        // in prior versions, numTries = numRetries
-        conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
         return ServerUtil.getDelegateHTableFactory(env, conf);
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5ff900e5/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
index cd196d3..aba2678 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
@@ -36,7 +36,6 @@ import org.apache.phoenix.hbase.index.table.HTableFactory;
 import org.apache.phoenix.hbase.index.table.HTableInterfaceReference;
 import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
 import org.apache.phoenix.index.PhoenixIndexFailurePolicy;
-import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.util.IndexUtil;
 
 import com.google.common.collect.Multimap;
@@ -58,8 +57,7 @@ public class ParallelWriterIndexCommitter implements IndexCommitter {
     public static final String INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY = "index.writer.threads.keepalivetime";
     private static final Log LOG = LogFactory.getLog(ParallelWriterIndexCommitter.class);
 
-    private HTableFactory retryingFactory;
-    private HTableFactory noRetriesfactory;
+    private HTableFactory factory;
     private Stoppable stopped;
     private QuickFailingTaskRunner pool;
     private KeyValueBuilder kvBuilder;
@@ -82,7 +80,6 @@ public class ParallelWriterIndexCommitter implements IndexCommitter {
                                 DEFAULT_CONCURRENT_INDEX_WRITER_THREADS).setCoreTimeout(
                                 INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY), env), env.getRegionServerServices(), parent, env);
         this.kvBuilder = KeyValueBuilder.get(env.getHBaseVersion());
-        this.noRetriesfactory = IndexWriterUtils.getNoRetriesHTableFactory(env);
     }
 
     /**
@@ -91,7 +88,7 @@ public class ParallelWriterIndexCommitter implements IndexCommitter {
      * Exposed for TESTING
      */
     void setup(HTableFactory factory, ExecutorService pool, Abortable abortable, Stoppable stop, RegionCoprocessorEnvironment env) {
-        this.retryingFactory = factory;
+        this.factory = factory;
         this.pool = new QuickFailingTaskRunner(pool);
         this.stopped = stop;
     }
@@ -165,8 +162,6 @@ public class ParallelWriterIndexCommitter implements IndexCommitter {
                                 }
                             }
                         }
-                     // if the client can retry index writes, then we don't need to retry here
-                        HTableFactory factory = clientVersion < PhoenixDatabaseMetaData.MIN_CLIENT_RETRY_INDEX_WRITES ? retryingFactory : noRetriesfactory;
                         table = factory.getTable(tableReference.get());
                         throwFailureIfDone();
                         table.batch(mutations);
@@ -231,8 +226,7 @@ public class ParallelWriterIndexCommitter implements IndexCommitter {
     public void stop(String why) {
         LOG.info("Shutting down " + this.getClass().getSimpleName() + " because " + why);
         this.pool.stop(why);
-        this.retryingFactory.shutdown();
-        this.noRetriesfactory.shutdown();
+        this.factory.shutdown();
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5ff900e5/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
index 1727fff..4dbad63 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/TrackingParallelWriterIndexCommitter.java
@@ -41,7 +41,6 @@ import org.apache.phoenix.hbase.index.table.HTableFactory;
 import org.apache.phoenix.hbase.index.table.HTableInterfaceReference;
 import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
 import org.apache.phoenix.index.PhoenixIndexFailurePolicy;
-import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.util.IndexUtil;
 
 import com.google.common.collect.Multimap;
@@ -75,8 +74,7 @@ public class TrackingParallelWriterIndexCommitter implements IndexCommitter {
     private static final String INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY = "index.writer.threads.keepalivetime";
 
     private TaskRunner pool;
-    private HTableFactory retryingFactory;
-    private HTableFactory noRetriesFactory;
+    private HTableFactory factory;
     private CapturingAbortable abortable;
     private Stoppable stopped;
     private RegionCoprocessorEnvironment env;
@@ -100,7 +98,6 @@ public class TrackingParallelWriterIndexCommitter implements IndexCommitter {
                                 DEFAULT_CONCURRENT_INDEX_WRITER_THREADS).setCoreTimeout(
                                 INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY), env), env.getRegionServerServices(), parent, env);
         this.kvBuilder = KeyValueBuilder.get(env.getHBaseVersion());
-        this.noRetriesFactory = IndexWriterUtils.getNoRetriesHTableFactory(env);
     }
 
     /**
@@ -111,7 +108,7 @@ public class TrackingParallelWriterIndexCommitter implements IndexCommitter {
     void setup(HTableFactory factory, ExecutorService pool, Abortable abortable, Stoppable stop,
             RegionCoprocessorEnvironment env) {
         this.pool = new WaitForCompletionTaskRunner(pool);
-        this.retryingFactory = factory;
+        this.factory = factory;
         this.abortable = new CapturingAbortable(abortable);
         this.stopped = stop;
         this.env = env;
@@ -178,8 +175,7 @@ public class TrackingParallelWriterIndexCommitter implements IndexCommitter {
                         if (LOG.isTraceEnabled()) {
                             LOG.trace("Writing index update:" + mutations + " to table: " + tableReference);
                         }
-                        // if the client can retry index writes, then we don't need to retry here
-                        HTableFactory factory = clientVersion < PhoenixDatabaseMetaData.MIN_CLIENT_RETRY_INDEX_WRITES ? retryingFactory : noRetriesFactory;
+
                         table = factory.getTable(tableReference.get());
                         throwFailureIfDone();
                         table.batch(mutations);
@@ -242,8 +238,7 @@ public class TrackingParallelWriterIndexCommitter implements IndexCommitter {
     public void stop(String why) {
         LOG.info("Shutting down " + this.getClass().getSimpleName());
         this.pool.stop(why);
-        this.retryingFactory.shutdown();
-        this.noRetriesFactory.shutdown();
+        this.factory.shutdown();
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5ff900e5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index d56628a..add0628 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -327,7 +327,6 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
     public static final int MIN_NAMESPACE_MAPPED_PHOENIX_VERSION = VersionUtil.encodeVersion("4", "8", "0");
     public static final int MIN_PENDING_ACTIVE_INDEX = VersionUtil.encodeVersion("4", "12", "0");
     public static final int MIN_PENDING_DISABLE_INDEX = VersionUtil.encodeVersion("4", "14", "0");
-    public static final int MIN_CLIENT_RETRY_INDEX_WRITES = VersionUtil.encodeVersion("4", "14", "0");
     public static final int MIN_TX_CLIENT_SIDE_MAINTENANCE = VersionUtil.encodeVersion("4", "14", "0");
     
     // Version below which we should turn off essential column family.


[23/50] [abbrv] phoenix git commit: Keep Ring buffer size small by default and avoid starting disruptor for server connection

Posted by td...@apache.org.
Keep Ring buffer size small by default and avoid starting disruptor for server connection


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/34f93d77
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/34f93d77
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/34f93d77

Branch: refs/heads/system-catalog
Commit: 34f93d77a37ca7c1e6ac96c2dc7970aafe08b960
Parents: db0e1ed
Author: Ankit Singhal <an...@gmail.com>
Authored: Tue Apr 17 15:08:20 2018 +0530
Committer: Ankit Singhal <an...@gmail.com>
Committed: Tue Apr 17 15:08:20 2018 +0530

----------------------------------------------------------------------
 .../org/apache/phoenix/log/QueryLoggerDisruptor.java |  2 +-
 .../phoenix/query/ConnectionQueryServicesImpl.java   | 15 +++++++++------
 .../main/java/org/apache/phoenix/util/QueryUtil.java | 14 ++++++++++++--
 3 files changed, 22 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/34f93d77/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerDisruptor.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerDisruptor.java b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerDisruptor.java
index b548d6c..1f2240e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerDisruptor.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/log/QueryLoggerDisruptor.java
@@ -43,7 +43,7 @@ public class QueryLoggerDisruptor implements Closeable{
     private volatile Disruptor<RingBufferEvent> disruptor;
     private boolean isClosed = false;
     //number of elements to create within the ring buffer.
-    private static final int RING_BUFFER_SIZE = 256 * 1024;
+    private static final int RING_BUFFER_SIZE = 8 * 1024;
     private static final Log LOG = LogFactory.getLog(QueryLoggerDisruptor.class);
     private static final String DEFAULT_WAIT_STRATEGY = BlockingWaitStrategy.class.getName();
     

http://git-wip-us.apache.org/repos/asf/phoenix/blob/34f93d77/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 5985705..f5e83f2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -195,7 +195,6 @@ import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
 import org.apache.phoenix.schema.EmptySequenceCacheException;
 import org.apache.phoenix.schema.FunctionNotFoundException;
 import org.apache.phoenix.schema.MetaDataClient;
-import org.apache.phoenix.schema.MetaDataSplitPolicy;
 import org.apache.phoenix.schema.NewerSchemaAlreadyExistsException;
 import org.apache.phoenix.schema.NewerTableAlreadyExistsException;
 import org.apache.phoenix.schema.PColumn;
@@ -243,6 +242,7 @@ import org.apache.phoenix.util.PhoenixContextExecutor;
 import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.PhoenixStopWatch;
 import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
 import org.apache.phoenix.util.ReadOnlyProps;
 import org.apache.phoenix.util.SchemaUtil;
 import org.apache.phoenix.util.ServerUtil;
@@ -406,11 +406,14 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         this.maxConnectionsAllowed = config.getInt(QueryServices.CLIENT_CONNECTION_MAX_ALLOWED_CONNECTIONS,
             QueryServicesOptions.DEFAULT_CLIENT_CONNECTION_MAX_ALLOWED_CONNECTIONS);
         this.shouldThrottleNumConnections = (maxConnectionsAllowed > 0);
-        try {
-            this.queryDisruptor = new QueryLoggerDisruptor(this.config);
-        } catch (SQLException e) {
-            logger.warn("Unable to initiate qeuery logging service !!");
-            e.printStackTrace();
+        if (!QueryUtil.isServerConnection(props)) {
+            //Start queryDistruptor everytime as log level can be change at connection level as well, but we can avoid starting for server connections.
+            try {
+                this.queryDisruptor = new QueryLoggerDisruptor(this.config);
+            } catch (SQLException e) {
+                logger.warn("Unable to initiate qeuery logging service !!");
+                e.printStackTrace();
+            }
         }
 
     }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/34f93d77/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java
index d7154a1..9d2e53c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java
@@ -82,6 +82,7 @@ public final class QueryUtil {
      */
     public static final int DATA_TYPE_NAME_POSITION = 6;
 
+    public static final String IS_SERVER_CONNECTION = "IS_SERVER_CONNECTION";
     private static final String SELECT = "SELECT";
     private static final String FROM = "FROM";
     private static final String WHERE = "WHERE";
@@ -358,6 +359,15 @@ public final class QueryUtil {
             SQLException {
         return getConnectionOnServer(new Properties(), conf);
     }
+    
+    public static void setServerConnection(Properties props){
+        UpgradeUtil.doNotUpgradeOnFirstConnection(props);
+        props.setProperty(IS_SERVER_CONNECTION, Boolean.TRUE.toString());
+    }
+    
+    public static boolean isServerConnection(ReadOnlyProps props) {
+        return props.getBoolean(IS_SERVER_CONNECTION, false);
+    }
 
     /**
      * @return {@link PhoenixConnection} with {@value UpgradeUtil#DO_NOT_UPGRADE} set so that we don't initiate metadata upgrade.
@@ -365,13 +375,13 @@ public final class QueryUtil {
     public static Connection getConnectionOnServer(Properties props, Configuration conf)
             throws ClassNotFoundException,
             SQLException {
-        UpgradeUtil.doNotUpgradeOnFirstConnection(props);
+        setServerConnection(props);
         return getConnection(props, conf);
     }
 
     public static Connection getConnectionOnServerWithCustomUrl(Properties props, String principal)
             throws SQLException, ClassNotFoundException {
-        UpgradeUtil.doNotUpgradeOnFirstConnection(props);
+        setServerConnection(props);
         String url = getConnectionUrl(props, null, principal);
         LOG.info("Creating connection with the jdbc url: " + url);
         return DriverManager.getConnection(url, props);


[10/50] [abbrv] phoenix git commit: PHOENIX-4366 Rebuilding a local index fails sometimes (addendum)

Posted by td...@apache.org.
PHOENIX-4366 Rebuilding a local index fails sometimes (addendum)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ca58fae2
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ca58fae2
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ca58fae2

Branch: refs/heads/system-catalog
Commit: ca58fae2037b0da9331aa97d9836ca195f289bb1
Parents: 5a16bff
Author: James Taylor <jt...@salesforce.com>
Authored: Wed Apr 11 15:20:08 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Apr 11 15:22:00 2018 -0700

----------------------------------------------------------------------
 .../NonAggregateRegionScannerFactory.java       | 37 +++++++++++---------
 1 file changed, 20 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/ca58fae2/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
index da99ff5..ba6a08f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
@@ -18,8 +18,15 @@
 
 package org.apache.phoenix.iterate;
 
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import static org.apache.phoenix.util.EncodedColumnsUtil.getMinMaxQualifiersFromScan;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
 
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.KeyValue;
@@ -61,16 +68,8 @@ import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.ServerUtil;
 
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-
-import static org.apache.phoenix.util.EncodedColumnsUtil.getMinMaxQualifiersFromScan;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 
 public class NonAggregateRegionScannerFactory extends RegionScannerFactory {
 
@@ -103,13 +102,17 @@ public class NonAggregateRegionScannerFactory extends RegionScannerFactory {
     boolean useNewValueColumnQualifier = EncodedColumnsUtil.useNewValueColumnQualifier(scan);
 
     Set<KeyValueColumnExpression> arrayKVRefs = Sets.newHashSet();
+    KeyValueSchema kvSchema = null;
+    ValueBitSet kvSchemaBitSet = null;
     Expression[] arrayFuncRefs = deserializeArrayPositionalExpressionInfoFromScan(scan, innerScanner, arrayKVRefs);
-    KeyValueSchema.KeyValueSchemaBuilder builder = new KeyValueSchema.KeyValueSchemaBuilder(0);
-    for (Expression expression : arrayFuncRefs) {
-        builder.addField(expression);
+    if (arrayFuncRefs != null) {
+        KeyValueSchema.KeyValueSchemaBuilder builder = new KeyValueSchema.KeyValueSchemaBuilder(0);
+        for (Expression expression : arrayFuncRefs) {
+            builder.addField(expression);
+        }
+        kvSchema = builder.build();
+        kvSchemaBitSet = ValueBitSet.newInstance(kvSchema);
     }
-    KeyValueSchema kvSchema = builder.build();
-    ValueBitSet kvSchemaBitSet = ValueBitSet.newInstance(kvSchema);
     TupleProjector tupleProjector = null;
     Region dataRegion = null;
     IndexMaintainer indexMaintainer = null;


[33/50] [abbrv] phoenix git commit: Merge remote-tracking branch 'upstream/master' into PHOENIX-3534

Posted by td...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 4d6075b,aba9600..21f4f74
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@@ -318,14 -359,12 +359,17 @@@ public class ConnectionlessQueryService
                  }
                  
                  try {
-                    metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_FUNCTION_METADATA);
+                     metaConnection.createStatement().executeUpdate(getFunctionTableDDL());
                  } catch (NewerTableAlreadyExistsException ignore) {
                  }
-                 
                  try {
-                     metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_CHILD_LINK_METADATA);
-                  } catch (NewerTableAlreadyExistsException ignore) {
-                  }
+                     metaConnection.createStatement().executeUpdate(getLogTableDDL());
+                 } catch (NewerTableAlreadyExistsException ignore) {}
++                try {
++                    metaConnection.createStatement()
++                            .executeUpdate(QueryConstants.CREATE_CHILD_LINK_METADATA);
++                } catch (NewerTableAlreadyExistsException ignore) {
++                }
              } catch (SQLException e) {
                  sqlE = e;
              } finally {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index aed22cd,22fa2f4..c7fa438
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@@ -18,98 -18,112 +18,7 @@@
  package org.apache.phoenix.query;
  
  
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.AUTO_PARTITION_SEQ;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BASE_COLUMN_COUNT;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BUFFER_LENGTH;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CACHE_SIZE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CHAR_OCTET_LENGTH;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLASS_NAME;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CURRENT_VALUE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CYCLE_FLAG;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ENCODING_SCHEME;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POST_KEY;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INCREMENT_BY;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_ARRAY;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_AUTOINCREMENT;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_CONSTANT;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_NAMESPACE_MAPPED;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_NULLABLE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_ROW_TIMESTAMP;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_VIEW_REFERENCED;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.JAR_PATH;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.KEY_SEQ;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LAST_STATS_UPDATE_TIME;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LIMIT_REACHED_FLAG;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MAX_VALUE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MIN_VALUE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MULTI_TENANT;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NULLABLE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_ARGS;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_PREC_RADIX;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PHYSICAL_NAME;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PK_NAME;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.REF_GENERATION;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.REMARKS;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.RETURN_TYPE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SALT_BUCKETS;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_CATALOG;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_SCHEMA;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_TABLE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SELF_REFERENCING_COL_NAME;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SEQUENCE_NAME;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SEQUENCE_SCHEMA;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SOURCE_DATA_TYPE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATA_TYPE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATETIME_SUB;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_WITH;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STORE_NULLS;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_TABLE;
- import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_TABLE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTION_PROVIDER;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_NAME;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
--import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BIND_PARAMETERS;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLIENT_IP;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXCEPTION_TRACE;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXPLAIN_PLAN;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GLOBAL_SCAN_DETAILS;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NO_OF_RESULTS_ITERATED;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_ID;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_STATUS;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCAN_METRICS_JSON;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_TIME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TOTAL_EXECUTION_TIME;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USER;
 -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE;
++import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.*;
  
  import java.math.BigDecimal;
  

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnImpl.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnImpl.java
index 76160ba,45aca98..7551b4c
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnImpl.java
@@@ -180,22 -137,8 +180,18 @@@ public class PColumnImpl implements PCo
      }
  
      @Override
 +    public long getTimestamp() {
 +        return timestamp;
 +    }
 +
 +    @Override
 +    public boolean isExcluded() {
 +        return dataType == null;
 +    }
 +
 +    @Override
      public boolean isNullable() {
-         // Only PK columns can be NOT NULL. We prevent this in the
-         // CREATE TABLE statement now (PHOENIX-1266), but this extra
-         // check for familyName != null will ensure that for existing
-         // tables we never treat key value columns as NOT NULL.
-         return nullable || familyName != null;
+         return nullable;
      }
  
      @Override

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index 186d4f8,082a58b..fbd7b7a
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@@ -296,21 -291,9 +296,21 @@@ public class PTableImpl implements PTab
                  table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(),
                  table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
                  table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
-                 table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(),
+                 table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.getTransactionProvider(), table.getUpdateCacheFrequency(),
                  table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
      }
 +
 +    // for views, the basePTable is for attributes we inherit from the physical table
 +    public static PTableImpl makePTable(PTable table, PTable basePTable, Collection<PColumn> columns, long timestamp, int baseTableColumnCount) throws SQLException {
 +        return new PTableImpl(
 +            table.getTenantId(), table.getSchemaName(), table.getTableName(), table.getType(), table.getIndexState(), timestamp,
 +            table.getSequenceNumber(), table.getPKName(), basePTable.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(),
 +            table.getIndexes(), basePTable.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
 +            table.isWALDisabled(), basePTable.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
 +            baseTableColumnCount, table.rowKeyOrderOptimizable(), basePTable.isTransactional(), table.getUpdateCacheFrequency(),
 +            table.getIndexDisableTimestamp(), basePTable.isNamespaceMapped(), basePTable.getAutoPartitionSeqName(), basePTable.isAppendOnlySchema(),
 +            basePTable.getImmutableStorageScheme(), basePTable.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
 +    }
      
      public static PTableImpl makePTable(PTable table, PTableType type, Collection<PColumn> columns) throws SQLException {
          return new PTableImpl(

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-protocol/src/main/MetaDataService.proto
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-protocol/src/main/PTable.proto
----------------------------------------------------------------------


[09/50] [abbrv] phoenix git commit: PHOENIX-4366 Rebuilding a local index fails sometimes

Posted by td...@apache.org.
PHOENIX-4366 Rebuilding a local index fails sometimes


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/5a16bff9
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/5a16bff9
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/5a16bff9

Branch: refs/heads/system-catalog
Commit: 5a16bff9f9eb9b97da5a3336cbf009b61d4deb2f
Parents: 1a312a9
Author: James Taylor <jt...@salesforce.com>
Authored: Wed Apr 11 13:37:45 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Apr 11 13:40:37 2018 -0700

----------------------------------------------------------------------
 .../coprocessor/BaseScannerRegionObserver.java  |  7 +----
 .../GroupedAggregateRegionObserver.java         |  4 +++
 .../phoenix/coprocessor/ScanRegionObserver.java |  4 ++-
 .../UngroupedAggregateRegionObserver.java       |  7 ++++-
 .../NonAggregateRegionScannerFactory.java       | 29 +++++++++-----------
 .../phoenix/iterate/RegionScannerFactory.java   |  4 +--
 .../apache/phoenix/iterate/SnapshotScanner.java |  6 +---
 7 files changed, 30 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a16bff9/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index 4a2c465..110a4ed 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -140,8 +140,6 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
 
     /** Exposed for testing */
     public static final String SCANNER_OPENED_TRACE_INFO = "Scanner opened on server";
-    protected QualifierEncodingScheme encodingScheme;
-    protected boolean useNewValueColumnQualifier;
 
     @Override
     public void start(CoprocessorEnvironment e) throws IOException {
@@ -212,8 +210,6 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
             // start exclusive and the stop inclusive.
             ScanUtil.setupReverseScan(scan);
         }
-        this.encodingScheme = EncodedColumnsUtil.getQualifierEncodingScheme(scan);
-        this.useNewValueColumnQualifier = EncodedColumnsUtil.useNewValueColumnQualifier(scan);
         return s;
     }
 
@@ -351,8 +347,7 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
             final byte[][] viewConstants, final TupleProjector projector,
             final ImmutableBytesWritable ptr, final boolean useQualiferAsListIndex) {
 
-        RegionScannerFactory regionScannerFactory = new NonAggregateRegionScannerFactory(c.getEnvironment(),
-            useNewValueColumnQualifier, encodingScheme);
+        RegionScannerFactory regionScannerFactory = new NonAggregateRegionScannerFactory(c.getEnvironment());
 
         return regionScannerFactory.getWrappedScanner(c.getEnvironment(), s, null, null, offset, scan, dataColumns, tupleProjector,
                 dataRegion, indexMaintainer, null, viewConstants, null, null, projector, ptr, useQualiferAsListIndex);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a16bff9/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
index 67cc114..201bcec 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
@@ -62,6 +62,7 @@ import org.apache.phoenix.index.IndexMaintainer;
 import org.apache.phoenix.join.HashJoinInfo;
 import org.apache.phoenix.memory.MemoryManager.MemoryChunk;
 import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.SortOrder;
 import org.apache.phoenix.schema.tuple.EncodedColumnQualiferCellsList;
 import org.apache.phoenix.schema.tuple.MultiKeyValueTuple;
@@ -110,6 +111,7 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver {
             keyOrdered = true;
         }
         int offset = 0;
+        boolean useNewValueColumnQualifier = EncodedColumnsUtil.useNewValueColumnQualifier(scan);
         if (ScanUtil.isLocalIndex(scan)) {
             /*
              * For local indexes, we need to set an offset on row key expressions to skip
@@ -395,6 +397,7 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver {
         boolean useQualifierAsIndex = EncodedColumnsUtil.useQualifierAsIndex(EncodedColumnsUtil.getMinMaxQualifiersFromScan(scan));
         final boolean spillableEnabled =
                 conf.getBoolean(GROUPBY_SPILLABLE_ATTRIB, DEFAULT_GROUPBY_SPILLABLE);
+        final PTable.QualifierEncodingScheme encodingScheme = EncodedColumnsUtil.getQualifierEncodingScheme(scan);
 
         GroupByCache groupByCache =
                 GroupByCacheFactory.INSTANCE.newCache(
@@ -466,6 +469,7 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver {
         }
         final Pair<Integer, Integer> minMaxQualifiers = EncodedColumnsUtil.getMinMaxQualifiersFromScan(scan);
         final boolean useQualifierAsIndex = EncodedColumnsUtil.useQualifierAsIndex(minMaxQualifiers);
+        final PTable.QualifierEncodingScheme encodingScheme = EncodedColumnsUtil.getQualifierEncodingScheme(scan);
         return new BaseRegionScanner(scanner) {
             private long rowCount = 0;
             private ImmutableBytesPtr currentKey = null;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a16bff9/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java
index b006ef6..2d9cd4f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.phoenix.expression.OrderByExpression;
 import org.apache.phoenix.iterate.NonAggregateRegionScannerFactory;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.util.EncodedColumnsUtil;
 
 /**
  *
@@ -68,7 +70,7 @@ public class ScanRegionObserver extends BaseScannerRegionObserver {
 
     @Override
     protected RegionScanner doPostScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c, final Scan scan, final RegionScanner s) throws Throwable {
-        NonAggregateRegionScannerFactory nonAggregateROUtil = new NonAggregateRegionScannerFactory(c.getEnvironment(), useNewValueColumnQualifier, encodingScheme);
+        NonAggregateRegionScannerFactory nonAggregateROUtil = new NonAggregateRegionScannerFactory(c.getEnvironment());
         return nonAggregateROUtil.getRegionScanner(scan, s);
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a16bff9/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 27d3880..de57772 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -376,7 +376,12 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
                     env, region.getRegionInfo().getTable().getNameAsString(), ts,
                     gp_width_bytes, gp_per_region_bytes);
             return collectStats(s, statsCollector, region, scan, env.getConfiguration());
-        } else if (ScanUtil.isIndexRebuild(scan)) { return rebuildIndices(s, region, scan, env.getConfiguration()); }
+        } else if (ScanUtil.isIndexRebuild(scan)) {
+            return rebuildIndices(s, region, scan, env.getConfiguration());
+        }
+
+        PTable.QualifierEncodingScheme encodingScheme = EncodedColumnsUtil.getQualifierEncodingScheme(scan);
+        boolean useNewValueColumnQualifier = EncodedColumnsUtil.useNewValueColumnQualifier(scan);
         int offsetToBe = 0;
         if (localIndexScan) {
             /*

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a16bff9/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
index c097d0d..da99ff5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
@@ -74,20 +74,13 @@ import static org.apache.phoenix.util.EncodedColumnsUtil.getMinMaxQualifiersFrom
 
 public class NonAggregateRegionScannerFactory extends RegionScannerFactory {
 
-  private ImmutableBytesWritable ptr = new ImmutableBytesWritable();
-  private KeyValueSchema kvSchema = null;
-  private ValueBitSet kvSchemaBitSet;
-
-  public NonAggregateRegionScannerFactory(RegionCoprocessorEnvironment env, boolean useNewValueColumnQualifier,
-      PTable.QualifierEncodingScheme encodingScheme) {
+  public NonAggregateRegionScannerFactory(RegionCoprocessorEnvironment env) {
     this.env = env;
-    this.useNewValueColumnQualifier = useNewValueColumnQualifier;
-    this.encodingScheme = encodingScheme;
   }
 
   @Override
   public RegionScanner getRegionScanner(final Scan scan, final RegionScanner s) throws Throwable {
-
+      ImmutableBytesWritable ptr = new ImmutableBytesWritable();
     int offset = 0;
     if (ScanUtil.isLocalIndex(scan)) {
             /*
@@ -106,9 +99,17 @@ public class NonAggregateRegionScannerFactory extends RegionScannerFactory {
       scanOffset = (Integer)PInteger.INSTANCE.toObject(scanOffsetBytes);
     }
     RegionScanner innerScanner = s;
+    PTable.QualifierEncodingScheme encodingScheme = EncodedColumnsUtil.getQualifierEncodingScheme(scan);
+    boolean useNewValueColumnQualifier = EncodedColumnsUtil.useNewValueColumnQualifier(scan);
 
     Set<KeyValueColumnExpression> arrayKVRefs = Sets.newHashSet();
-    Expression[] arrayFuncRefs = deserializeArrayPostionalExpressionInfoFromScan(scan, innerScanner, arrayKVRefs);
+    Expression[] arrayFuncRefs = deserializeArrayPositionalExpressionInfoFromScan(scan, innerScanner, arrayKVRefs);
+    KeyValueSchema.KeyValueSchemaBuilder builder = new KeyValueSchema.KeyValueSchemaBuilder(0);
+    for (Expression expression : arrayFuncRefs) {
+        builder.addField(expression);
+    }
+    KeyValueSchema kvSchema = builder.build();
+    ValueBitSet kvSchemaBitSet = ValueBitSet.newInstance(kvSchema);
     TupleProjector tupleProjector = null;
     Region dataRegion = null;
     IndexMaintainer indexMaintainer = null;
@@ -196,13 +197,12 @@ public class NonAggregateRegionScannerFactory extends RegionScannerFactory {
     }
   }
 
-  private Expression[] deserializeArrayPostionalExpressionInfoFromScan(Scan scan, RegionScanner s,
-      Set<KeyValueColumnExpression> arrayKVRefs) {
+  private Expression[] deserializeArrayPositionalExpressionInfoFromScan(Scan scan, RegionScanner s,
+                                                                        Set<KeyValueColumnExpression> arrayKVRefs) {
     byte[] specificArrayIdx = scan.getAttribute(BaseScannerRegionObserver.SPECIFIC_ARRAY_INDEX);
     if (specificArrayIdx == null) {
       return null;
     }
-    KeyValueSchema.KeyValueSchemaBuilder builder = new KeyValueSchema.KeyValueSchemaBuilder(0);
     ByteArrayInputStream stream = new ByteArrayInputStream(specificArrayIdx);
     try {
       DataInputStream input = new DataInputStream(stream);
@@ -220,10 +220,7 @@ public class NonAggregateRegionScannerFactory extends RegionScannerFactory {
         ArrayIndexFunction arrayIdxFunc = new ArrayIndexFunction();
         arrayIdxFunc.readFields(input);
         arrayFuncRefs[i] = arrayIdxFunc;
-        builder.addField(arrayIdxFunc);
       }
-      kvSchema = builder.build();
-      kvSchemaBitSet = ValueBitSet.newInstance(kvSchema);
       return arrayFuncRefs;
     } catch (IOException e) {
       throw new RuntimeException(e);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a16bff9/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java
index 3dcbef9..aed5805 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java
@@ -41,6 +41,7 @@ import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.ValueBitSet;
 import org.apache.phoenix.schema.tuple.*;
 import org.apache.phoenix.transaction.PhoenixTransactionContext;
+import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.ServerUtil;
@@ -54,8 +55,6 @@ import java.util.Set;
 public abstract class RegionScannerFactory {
 
   protected RegionCoprocessorEnvironment env;
-  protected boolean useNewValueColumnQualifier;
-  protected PTable.QualifierEncodingScheme encodingScheme;
 
   /**
    * Returns the region based on the value of the
@@ -107,6 +106,7 @@ public abstract class RegionScannerFactory {
       private boolean hasReferences = checkForReferenceFiles();
       private HRegionInfo regionInfo = env.getRegionInfo();
       private byte[] actualStartKey = getActualStartKey();
+      private boolean useNewValueColumnQualifier = EncodedColumnsUtil.useNewValueColumnQualifier(scan);
 
       // If there are any reference files after local index region merge some cases we might
       // get the records less than scan start row key. This will happen when we replace the

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a16bff9/phoenix-core/src/main/java/org/apache/phoenix/iterate/SnapshotScanner.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SnapshotScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SnapshotScanner.java
index 46329a2..305d558 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SnapshotScanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SnapshotScanner.java
@@ -55,15 +55,11 @@ public class SnapshotScanner extends AbstractClientScanner {
     values = new ArrayList<>();
     this.region = HRegion.openHRegion(conf, fs, rootDir, hri, htd, null, null, null);
 
-    // process the region scanner for non-aggregate queries
-    PTable.QualifierEncodingScheme encodingScheme = EncodedColumnsUtil.getQualifierEncodingScheme(scan);
-    boolean useNewValueColumnQualifier = EncodedColumnsUtil.useNewValueColumnQualifier(scan);
-
     RegionCoprocessorEnvironment snapshotEnv = getSnapshotContextEnvironment(conf);
 
     RegionScannerFactory regionScannerFactory;
     if (scan.getAttribute(BaseScannerRegionObserver.NON_AGGREGATE_QUERY) != null) {
-      regionScannerFactory = new NonAggregateRegionScannerFactory(snapshotEnv, useNewValueColumnQualifier, encodingScheme);
+      regionScannerFactory = new NonAggregateRegionScannerFactory(snapshotEnv);
     } else {
       /* future work : Snapshot M/R jobs for aggregate queries*/
       throw new UnsupportedOperationException("Snapshot M/R jobs not available for aggregate queries");


[11/50] [abbrv] phoenix git commit: PHOENIX-4605 Support running multiple transaction providers

Posted by td...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionTable.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionTable.java
deleted file mode 100644
index 55785be..0000000
--- a/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionTable.java
+++ /dev/null
@@ -1,366 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.transaction;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Append;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Increment;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Row;
-import org.apache.hadoop.hbase.client.RowMutations;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.coprocessor.Batch.Call;
-import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
-import org.apache.tephra.TxConstants;
-import org.apache.tephra.hbase.TransactionAwareHTable;
-import org.apache.phoenix.schema.PTable;
-import org.apache.phoenix.schema.PTableType;
-
-import com.google.protobuf.Descriptors.MethodDescriptor;
-import com.google.protobuf.Message;
-import com.google.protobuf.Service;
-import com.google.protobuf.ServiceException;
-
-public class TephraTransactionTable implements PhoenixTransactionalTable {
-
-    private TransactionAwareHTable transactionAwareHTable;
-
-    private TephraTransactionContext tephraTransactionContext;
-
-    public TephraTransactionTable(PhoenixTransactionContext ctx, HTableInterface hTable) {
-        this(ctx, hTable, null);
-    }
-
-    public TephraTransactionTable(PhoenixTransactionContext ctx, HTableInterface hTable, PTable pTable) {
-
-        assert(ctx instanceof TephraTransactionContext);
-
-        tephraTransactionContext = (TephraTransactionContext) ctx;
-
-        transactionAwareHTable = new TransactionAwareHTable(hTable, (pTable != null && pTable.isImmutableRows()) ? TxConstants.ConflictDetection.NONE : TxConstants.ConflictDetection.ROW);
-
-        tephraTransactionContext.addTransactionAware(transactionAwareHTable);
-
-        if (pTable != null && pTable.getType() != PTableType.INDEX) {
-            tephraTransactionContext.markDMLFence(pTable);
-        }
-    }
-
-    @Override
-    public Result get(Get get) throws IOException {
-        return transactionAwareHTable.get(get);
-    }
-
-    @Override
-    public void put(Put put) throws IOException {
-        transactionAwareHTable.put(put);
-    }
-
-    @Override
-    public void delete(Delete delete) throws IOException {
-        transactionAwareHTable.delete(delete);
-    }
-
-    @Override
-    public ResultScanner getScanner(Scan scan) throws IOException {
-        return transactionAwareHTable.getScanner(scan);
-    }
-
-    @Override
-    public byte[] getTableName() {
-        return transactionAwareHTable.getTableName();
-    }
-
-    @Override
-    public Configuration getConfiguration() {
-        return transactionAwareHTable.getConfiguration();
-    }
-
-    @Override
-    public HTableDescriptor getTableDescriptor() throws IOException {
-        return transactionAwareHTable.getTableDescriptor();
-    }
-
-    @Override
-    public boolean exists(Get get) throws IOException {
-        return transactionAwareHTable.exists(get);
-    }
-
-    @Override
-    public Result[] get(List<Get> gets) throws IOException {
-        return transactionAwareHTable.get(gets);
-    }
-
-    @Override
-    public ResultScanner getScanner(byte[] family) throws IOException {
-        return transactionAwareHTable.getScanner(family);
-    }
-
-    @Override
-    public ResultScanner getScanner(byte[] family, byte[] qualifier)
-            throws IOException {
-        return transactionAwareHTable.getScanner(family, qualifier);
-    }
-
-    @Override
-    public void put(List<Put> puts) throws IOException {
-        transactionAwareHTable.put(puts);
-    }
-
-    @Override
-    public void delete(List<Delete> deletes) throws IOException {
-        transactionAwareHTable.delete(deletes);
-    }
-
-    @Override
-    public void setAutoFlush(boolean autoFlush) {
-        transactionAwareHTable.setAutoFlush(autoFlush);
-    }
-
-    @Override
-    public boolean isAutoFlush() {
-        return transactionAwareHTable.isAutoFlush();
-    }
-
-    @Override
-    public long getWriteBufferSize() {
-        return transactionAwareHTable.getWriteBufferSize();
-    }
-
-    @Override
-    public void setWriteBufferSize(long writeBufferSize) throws IOException {
-        transactionAwareHTable.setWriteBufferSize(writeBufferSize);
-    }
-
-    @Override
-    public void flushCommits() throws IOException {
-        transactionAwareHTable.flushCommits();
-    }
-
-    @Override
-    public void close() throws IOException {
-        transactionAwareHTable.close();
-    }
-
-    @Override
-    public long incrementColumnValue(byte[] row, byte[] family,
-            byte[] qualifier, long amount, boolean writeToWAL)
-            throws IOException {
-        return transactionAwareHTable.incrementColumnValue(row, family, qualifier, amount, writeToWAL);
-    }
-
-    @Override
-    public Boolean[] exists(List<Get> gets) throws IOException {
-        return transactionAwareHTable.exists(gets);
-    }
-
-    @Override
-    public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
-        transactionAwareHTable.setAutoFlush(autoFlush, clearBufferOnFail);
-    }
-
-    @Override
-    public void setAutoFlushTo(boolean autoFlush) {
-        transactionAwareHTable.setAutoFlush(autoFlush);
-    }
-
-    @Override
-    public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
-        return transactionAwareHTable.getRowOrBefore(row, family);
-    }
-
-    @Override
-    public TableName getName() {
-        return transactionAwareHTable.getName();
-    }
-
-    @Override
-    public boolean[] existsAll(List<Get> gets) throws IOException {
-        return transactionAwareHTable.existsAll(gets);
-    }
-
-    @Override
-    public void batch(List<? extends Row> actions, Object[] results)
-            throws IOException, InterruptedException {
-        transactionAwareHTable.batch(actions, results);
-    }
-
-    @Override
-    public Object[] batch(List<? extends Row> actions) throws IOException,
-            InterruptedException {
-        return transactionAwareHTable.batch(actions);
-    }
-
-    @Override
-    public <R> void batchCallback(List<? extends Row> actions,
-            Object[] results, Callback<R> callback) throws IOException,
-            InterruptedException {
-        transactionAwareHTable.batchCallback(actions, results, callback);
-    }
-
-    @Override
-    public <R> Object[] batchCallback(List<? extends Row> actions,
-            Callback<R> callback) throws IOException, InterruptedException {
-        return transactionAwareHTable.batchCallback(actions, callback);
-    }
-
-    @Override
-    public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier,
-            byte[] value, Put put) throws IOException {
-        return transactionAwareHTable.checkAndPut(row, family, qualifier, value, put);
-    }
-
-    @Override
-    public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier,
-            CompareOp compareOp, byte[] value, Put put) throws IOException {
-        return transactionAwareHTable.checkAndPut(row, family, qualifier, compareOp, value, put);
-    }
-
-    @Override
-    public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
-            byte[] value, Delete delete) throws IOException {
-        return transactionAwareHTable.checkAndDelete(row, family, qualifier, value, delete);
-    }
-
-    @Override
-    public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
-            CompareOp compareOp, byte[] value, Delete delete)
-            throws IOException {
-        return transactionAwareHTable.checkAndDelete(row, family, qualifier, compareOp, value, delete);
-    }
-
-    @Override
-    public void mutateRow(RowMutations rm) throws IOException {
-        transactionAwareHTable.mutateRow(rm);
-    }
-
-    @Override
-    public Result append(Append append) throws IOException {
-        return transactionAwareHTable.append(append);
-    }
-
-    @Override
-    public Result increment(Increment increment) throws IOException {
-        return transactionAwareHTable.increment(increment);
-    }
-
-    @Override
-    public long incrementColumnValue(byte[] row, byte[] family,
-            byte[] qualifier, long amount) throws IOException {
-        return transactionAwareHTable.incrementColumnValue(row, family, qualifier, amount);
-    }
-
-    @Override
-    public long incrementColumnValue(byte[] row, byte[] family,
-            byte[] qualifier, long amount, Durability durability)
-            throws IOException {
-        return transactionAwareHTable.incrementColumnValue(row, family, qualifier, amount, durability);
-    }
-
-    @Override
-    public CoprocessorRpcChannel coprocessorService(byte[] row) {
-        return transactionAwareHTable.coprocessorService(row);
-    }
-
-    @Override
-    public <T extends Service, R> Map<byte[], R> coprocessorService(
-            Class<T> service, byte[] startKey, byte[] endKey,
-            Call<T, R> callable) throws ServiceException, Throwable {
-        return transactionAwareHTable.coprocessorService(service, startKey, endKey, callable);
-    }
-
-    @Override
-    public <T extends Service, R> void coprocessorService(Class<T> service,
-            byte[] startKey, byte[] endKey, Call<T, R> callable,
-            Callback<R> callback) throws ServiceException, Throwable {
-        transactionAwareHTable.coprocessorService(service, startKey, endKey, callable, callback);
-    }
-
-    @Override
-    public <R extends Message> Map<byte[], R> batchCoprocessorService(
-            MethodDescriptor methodDescriptor, Message request,
-            byte[] startKey, byte[] endKey, R responsePrototype)
-            throws ServiceException, Throwable {
-        return transactionAwareHTable.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype);
-    }
-
-    @Override
-    public <R extends Message> void batchCoprocessorService(
-            MethodDescriptor methodDescriptor, Message request,
-            byte[] startKey, byte[] endKey, R responsePrototype,
-            Callback<R> callback) throws ServiceException, Throwable {
-        transactionAwareHTable.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype, callback);
-    }
-
-    @Override
-    public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier,
-            CompareOp compareOp, byte[] value, RowMutations mutation)
-            throws IOException {
-        return transactionAwareHTable.checkAndMutate(row, family, qualifier, compareOp, value, mutation);
-    }
-
-    @Override
-    public int getOperationTimeout() {
-        return 0;
-    }
-
-    @Override
-    public void setOperationTimeout(int i) {
-    }
-
-    @Override
-    public int getRpcTimeout() {
-        return 0;
-    }
-
-    @Override
-    public void setRpcTimeout(int i) {
-    }
-
-    @Override
-    public int getReadRpcTimeout() {
-        return 0;
-    }
-
-    @Override
-    public void setReadRpcTimeout(int readRpcTimeout) {
-    }
-
-    @Override
-    public int getWriteRpcTimeout() {
-        return 0;
-    }
-
-    @Override
-    public void setWriteRpcTimeout(int writeRpcTimeout) {
-    }
-}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/transaction/TransactionFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/TransactionFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/TransactionFactory.java
index ea2822b..67c4bc1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/transaction/TransactionFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/transaction/TransactionFactory.java
@@ -17,25 +17,56 @@
  */
 package org.apache.phoenix.transaction;
 
+import java.io.IOException;
+
+import org.apache.phoenix.coprocessor.MetaDataProtocol;
+
+
 
 public class TransactionFactory {
-    enum TransactionProcessor {
-        Tephra,
-        Omid
+    public enum Provider {
+        TEPHRA((byte)1, TephraTransactionProvider.getInstance()),
+        OMID((byte)2, OmidTransactionProvider.getInstance());
+        
+        private final byte code;
+        private final PhoenixTransactionProvider provider;
+        
+        Provider(byte code, PhoenixTransactionProvider provider) {
+            this.code = code;
+            this.provider = provider;
+        }
+        
+        public byte getCode() {
+            return this.code;
+        }
+
+        public static Provider fromCode(int code) {
+            if (code < 1 || code > Provider.values().length) {
+                throw new IllegalArgumentException("Invalid TransactionFactory.Provider " + code);
+            }
+            return Provider.values()[code-1];
+        }
+        
+        public static Provider getDefault() {
+            return TEPHRA;
+        }
+
+        public PhoenixTransactionProvider getTransactionProvider()  {
+            return provider;
+        }
     }
 
-    static public TransactionProvider getTransactionProvider() {
-        return TephraTransactionProvider.getInstance();
+    public static PhoenixTransactionProvider getTransactionProvider(Provider provider) {
+        return provider.getTransactionProvider();
     }
     
-    static public TransactionProvider getTransactionProvider(TransactionProcessor processor) {
-        switch (processor) {
-        case Tephra:
-            return TephraTransactionProvider.getInstance();
-        case Omid:
-            return OmidTransactionProvider.getInstance();
-        default:
-            throw new IllegalArgumentException("Unknown transaction processor: " + processor);
+    public static PhoenixTransactionContext getTransactionContext(byte[] txState, int clientVersion) throws IOException {
+        if (txState == null || txState.length == 0) {
+            return null;
         }
+        Provider provider = (clientVersion < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0) 
+                ? Provider.OMID
+                : Provider.fromCode(txState[txState.length-1]);
+        return provider.getTransactionProvider().getTransactionContext(txState);
     }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/transaction/TransactionProvider.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/TransactionProvider.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/TransactionProvider.java
deleted file mode 100644
index a5704f1..0000000
--- a/phoenix-core/src/main/java/org/apache/phoenix/transaction/TransactionProvider.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.transaction;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.phoenix.jdbc.PhoenixConnection;
-
-public interface TransactionProvider {
-    public PhoenixTransactionContext getTransactionContext();
-    public PhoenixTransactionContext getTransactionContext(byte[] txnBytes) throws IOException;
-    public PhoenixTransactionContext getTransactionContext(PhoenixConnection connection);
-    public PhoenixTransactionContext getTransactionContext(PhoenixTransactionContext contex, PhoenixConnection connection, boolean subTask);
-    
-    public PhoenixTransactionalTable getTransactionalTable(PhoenixTransactionContext ctx, HTableInterface htable);
-    
-    public Cell newDeleteFamilyMarker(byte[] row, byte[] family, long timestamp);
-    public Cell newDeleteColumnMarker(byte[] row, byte[] family, byte[] qualifier, long timestamp);
-}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
index 1c25c33..6cf6e56 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
@@ -88,7 +88,6 @@ import org.apache.phoenix.schema.RowKeyValueAccessor;
 import org.apache.phoenix.schema.TableNotFoundException;
 import org.apache.phoenix.schema.ValueBitSet;
 import org.apache.phoenix.schema.types.PDataType;
-import org.apache.phoenix.transaction.TransactionFactory;
 
 import com.google.common.base.Function;
 import com.google.common.base.Joiner;
@@ -1515,7 +1514,7 @@ public class PhoenixRuntime {
      * @return wall clock time in milliseconds (i.e. Epoch time) of a given Cell time stamp.
      */
     public static long getWallClockTimeFromCellTimeStamp(long tsOfCell) {
-        return TransactionFactory.getTransactionProvider().getTransactionContext().isPreExistingVersion(tsOfCell) ? tsOfCell : TransactionUtil.convertToMilliseconds(tsOfCell);
+        return TransactionUtil.isTransactionalTimestamp(tsOfCell) ? TransactionUtil.convertToMilliseconds(tsOfCell) : tsOfCell;
     }
 
     public static long getCurrentScn(ReadOnlyProps props) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
index dd885fd..996e1dc 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -59,6 +59,7 @@ import org.apache.phoenix.filter.DistinctPrefixFilter;
 import org.apache.phoenix.filter.MultiEncodedCQKeyValueComparisonFilter;
 import org.apache.phoenix.filter.SkipScanFilter;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.hbase.index.util.VersionUtil;
 import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.query.KeyRange;
 import org.apache.phoenix.query.KeyRange.Bound;
@@ -88,6 +89,8 @@ import com.google.common.collect.Lists;
  */
 public class ScanUtil {
     public static final int[] SINGLE_COLUMN_SLOT_SPAN = new int[1];
+    public static final int UNKNOWN_CLIENT_VERSION = VersionUtil.encodeVersion(4, 4, 0);
+
     /*
      * Max length that we fill our key when we turn an inclusive key
      * into a exclusive key.
@@ -930,5 +933,17 @@ public class ScanUtil {
     public static boolean isIndexRebuild(Scan scan) {
         return scan.getAttribute((BaseScannerRegionObserver.REBUILD_INDEXES)) != null;
     }
+ 
+    public static int getClientVersion(Scan scan) {
+        int clientVersion = UNKNOWN_CLIENT_VERSION;
+        byte[] clientVersionBytes = scan.getAttribute(BaseScannerRegionObserver.CLIENT_VERSION);
+        if (clientVersionBytes != null) {
+            clientVersion = Bytes.toInt(clientVersionBytes);
+        }
+        return clientVersion;
+    }
     
+    public static void setClientVersion(Scan scan, int version) {
+        scan.setAttribute(BaseScannerRegionObserver.CLIENT_VERSION, Bytes.toBytes(version));
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java
index 8cedb1c..dee02d1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java
@@ -25,42 +25,57 @@ import java.util.Map;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
 import org.apache.phoenix.execute.MutationState;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.schema.PTable;
-import org.apache.phoenix.transaction.PhoenixTransactionContext;
-import org.apache.phoenix.transaction.PhoenixTransactionalTable;
-import org.apache.phoenix.transaction.TephraTransactionTable;
 import org.apache.phoenix.transaction.TransactionFactory;
-import org.apache.tephra.util.TxUtils;
 
 public class TransactionUtil {
+    // All transaction providers must use an empty byte array as the family delete marker
+    // (see TxConstants.FAMILY_DELETE_QUALIFIER)
+    public static final byte[] FAMILY_DELETE_MARKER = HConstants.EMPTY_BYTE_ARRAY;
+    // All transaction providers must multiply timestamps by this constant.
+    // (see TxConstants.MAX_TX_PER_MS)
+    public static final int MAX_TRANSACTIONS_PER_MILLISECOND = 1000000;
+    // Constant used to empirically determine if a timestamp is a transactional or
+    // non transactional timestamp (see TxUtils.MAX_NON_TX_TIMESTAMP)
+    private static final long MAX_NON_TX_TIMESTAMP = (long) (System.currentTimeMillis() * 1.1);
+    
     private TransactionUtil() {
+        
     }
     
     public static boolean isTransactionalTimestamp(long ts) {
-        return !TxUtils.isPreExistingVersion(ts);
+        return ts >= MAX_NON_TX_TIMESTAMP;
     }
     
     public static boolean isDelete(Cell cell) {
-        return (CellUtil.matchingValue(cell, HConstants.EMPTY_BYTE_ARRAY));
+        return CellUtil.matchingValue(cell, HConstants.EMPTY_BYTE_ARRAY);
     }
     
-    public static long convertToNanoseconds(long serverTimeStamp) {
-        return serverTimeStamp * TransactionFactory.getTransactionProvider().getTransactionContext().getMaxTransactionsPerSecond();
+    public static boolean isDeleteFamily(Cell cell) {
+        return CellUtil.matchingQualifier(cell, FAMILY_DELETE_MARKER) && CellUtil.matchingValue(cell, HConstants.EMPTY_BYTE_ARRAY);
     }
     
-    public static long convertToMilliseconds(long serverTimeStamp) {
-        return serverTimeStamp / TransactionFactory.getTransactionProvider().getTransactionContext().getMaxTransactionsPerSecond();
+    private static Cell newDeleteFamilyMarker(byte[] row, byte[] family, long timestamp) {
+        return CellUtil.createCell(row, family, FAMILY_DELETE_MARKER, timestamp, KeyValue.Type.Put.getCode(), HConstants.EMPTY_BYTE_ARRAY);
     }
     
-    public static PhoenixTransactionalTable getPhoenixTransactionTable(PhoenixTransactionContext phoenixTransactionContext, HTableInterface htable, PTable pTable) {
-        return new TephraTransactionTable(phoenixTransactionContext, htable, pTable);
+    private static Cell newDeleteColumnMarker(byte[] row, byte[] family, byte[] qualifier, long timestamp) {
+        return CellUtil.createCell(row, family, qualifier, timestamp, KeyValue.Type.Put.getCode(), HConstants.EMPTY_BYTE_ARRAY);
+    }
+
+    public static long convertToNanoseconds(long serverTimeStamp) {
+        return serverTimeStamp * MAX_TRANSACTIONS_PER_MILLISECOND;
+    }
+    
+    public static long convertToMilliseconds(long serverTimeStamp) {
+        return serverTimeStamp / MAX_TRANSACTIONS_PER_MILLISECOND;
     }
     
     // we resolve transactional tables at the txn read pointer
@@ -83,14 +98,14 @@ public class TransactionUtil {
 		return  txInProgress ? convertToMilliseconds(mutationState.getInitialWritePointer()) : result.getMutationTime();
 	}
 
-	public static Long getTableTimestamp(PhoenixConnection connection, boolean transactional) throws SQLException {
+	public static Long getTableTimestamp(PhoenixConnection connection, boolean transactional, TransactionFactory.Provider provider) throws SQLException {
 		Long timestamp = null;
 		if (!transactional) {
 			return timestamp;
 		}
 		MutationState mutationState = connection.getMutationState();
 		if (!mutationState.isTransactionStarted()) {
-			mutationState.startTransaction();
+			mutationState.startTransaction(provider);
 		}
 		timestamp = convertToMilliseconds(mutationState.getInitialWritePointer());
 		return timestamp;
@@ -108,7 +123,7 @@ public class TransactionUtil {
                         if (deleteMarker == null) {
                             deleteMarker = new Put(mutation.getRow());
                         }
-                        deleteMarker.add(TransactionFactory.getTransactionProvider().newDeleteFamilyMarker(
+                        deleteMarker.add(newDeleteFamilyMarker(
                                 deleteMarker.getRow(), 
                                 family, 
                                 familyCells.get(0).getTimestamp()));
@@ -119,7 +134,7 @@ public class TransactionUtil {
                             if (deleteMarker == null) {
                                 deleteMarker = new Put(mutation.getRow());
                             }
-                            deleteMarker.add(TransactionFactory.getTransactionProvider().newDeleteColumnMarker(
+                            deleteMarker.add(newDeleteColumnMarker(
                                     deleteMarker.getRow(),
                                     family,
                                     CellUtil.cloneQualifier(cell), 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
index 76757b0..d88a915 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
@@ -56,7 +56,6 @@ import org.apache.phoenix.jdbc.PhoenixStatement;
 import org.apache.phoenix.parse.JoinTableNode.JoinType;
 import org.apache.phoenix.parse.ParseNodeFactory;
 import org.apache.phoenix.parse.SelectStatement;
-import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.schema.ColumnRef;
 import org.apache.phoenix.schema.PColumn;
 import org.apache.phoenix.schema.PColumnImpl;
@@ -64,8 +63,8 @@ import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PNameFactory;
 import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTable.EncodedCQCounter;
-import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.PTableImpl;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.TableRef;
@@ -262,7 +261,7 @@ public class CorrelatePlanTest {
                     PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM,
                     null, null, columns, null, null, Collections.<PTable>emptyList(),
                     false, Collections.<PName>emptyList(), null, null, false, false, false, null,
-                    null, null, true, false, 0, 0L, Boolean.FALSE, null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, EncodedCQCounter.NULL_COUNTER, true);
+                    null, null, true, null, 0, 0L, Boolean.FALSE, null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, EncodedCQCounter.NULL_COUNTER, true);
             TableRef sourceTable = new TableRef(pTable);
             List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList();
             for (PColumn column : sourceTable.getTable().getColumns()) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
index 1a7132c..017e6c8 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
@@ -50,7 +50,6 @@ import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixStatement;
 import org.apache.phoenix.parse.ParseNodeFactory;
 import org.apache.phoenix.parse.SelectStatement;
-import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.schema.ColumnRef;
 import org.apache.phoenix.schema.PColumn;
 import org.apache.phoenix.schema.PColumnImpl;
@@ -58,11 +57,11 @@ import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PNameFactory;
 import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTable.EncodedCQCounter;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.PTableImpl;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.TableRef;
-import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 import org.apache.phoenix.schema.tuple.SingleKeyValueTuple;
 import org.apache.phoenix.schema.tuple.Tuple;
 import org.junit.Test;
@@ -183,7 +182,7 @@ public class LiteralResultIteratorPlanTest {
             PTable pTable = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME, PTableType.SUBQUERY, null,
                     MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null,
                     Collections.<PTable> emptyList(), false, Collections.<PName> emptyList(), null, null, false, false,
-                    false, null, null, null, true, false, 0, 0L, false, null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, EncodedCQCounter.NULL_COUNTER, true);
+                    false, null, null, null, true, null, 0, 0L, false, null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, EncodedCQCounter.NULL_COUNTER, true);
             TableRef sourceTable = new TableRef(pTable);
             List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList();
             for (PColumn column : sourceTable.getTable().getColumns()) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index 580becb..0ea63e7 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -130,7 +130,6 @@ import org.apache.phoenix.schema.NewerTableAlreadyExistsException;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.TableAlreadyExistsException;
 import org.apache.phoenix.schema.TableNotFoundException;
-import org.apache.phoenix.transaction.TransactionFactory;
 import org.apache.phoenix.util.ConfigUtil;
 import org.apache.phoenix.util.DateUtil;
 import org.apache.phoenix.util.PhoenixRuntime;
@@ -168,7 +167,6 @@ public abstract class BaseTest {
     
     private static final Map<String,String> tableDDLMap;
     private static final Logger logger = LoggerFactory.getLogger(BaseTest.class);
-    protected static final int DEFAULT_TXN_TIMEOUT_SECONDS = 30;
     @ClassRule
     public static TemporaryFolder tmpFolder = new TemporaryFolder();
     private static final int dropTableTimeout = 300; // 5 mins should be long enough.
@@ -414,18 +412,6 @@ public abstract class BaseTest {
         return url;
     }
     
-    private static void tearDownTxManager() throws SQLException {
-        TransactionFactory.getTransactionProvider().getTransactionContext().tearDownTxManager();
-    }
-
-    protected static void setTxnConfigs() throws IOException {
-        TransactionFactory.getTransactionProvider().getTransactionContext().setTxnConfigs(config, tmpFolder.newFolder().getAbsolutePath(), DEFAULT_TXN_TIMEOUT_SECONDS);
-    }
-
-    protected static void setupTxManager() throws SQLException, IOException {
-        TransactionFactory.getTransactionProvider().getTransactionContext().setupTxManager(config, getUrl());
-    }
-
     private static String checkClusterInitialized(ReadOnlyProps serverProps) throws Exception {
         if (!clusterInitialized) {
             url = setUpTestCluster(config, serverProps);
@@ -434,10 +420,6 @@ public abstract class BaseTest {
         return url;
     }
 
-    private static void checkTxManagerInitialized(ReadOnlyProps clientProps) throws SQLException, IOException {
-        setupTxManager();
-    }
-
     /**
      * Set up the test hbase cluster.
      * @return url to be used by clients to connect to the cluster.
@@ -476,11 +458,6 @@ public abstract class BaseTest {
         final HBaseTestingUtility u = utility;
         try {
             destroyDriver();
-            try {
-                tearDownTxManager();
-            } catch (Throwable t) {
-                logger.error("Exception caught when shutting down tx manager", t);
-            }
             utility = null;
             clusterInitialized = false;
         } finally {
@@ -519,9 +496,7 @@ public abstract class BaseTest {
     
     protected static void setUpTestDriver(ReadOnlyProps serverProps, ReadOnlyProps clientProps) throws Exception {
         if (driver == null) {
-            setTxnConfigs();
             String url = checkClusterInitialized(serverProps);
-            checkTxManagerInitialized(serverProps);
             driver = initAndRegisterTestDriver(url, clientProps);
         }
     }
@@ -593,6 +568,7 @@ public abstract class BaseTest {
         conf.set(RSRpcServices.REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS, DEFAULT_RPC_SCHEDULER_FACTORY);
         conf.setLong(HConstants.ZK_SESSION_TIMEOUT, 10 * HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
         conf.setLong(HConstants.ZOOKEEPER_TICK_TIME, 6 * 1000);
+        
         // override any defaults based on overrideProps
         for (Entry<String,String> entry : overrideProps) {
             conf.set(entry.getKey(), entry.getValue());

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
index c93e56e..a7569f7 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
@@ -20,9 +20,12 @@ package org.apache.phoenix.query;
 import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_SPOOL_DIRECTORY;
 import static org.apache.phoenix.query.QueryServicesOptions.withDefaults;
 
+import org.apache.curator.shaded.com.google.common.io.Files;
 import org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec;
 import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.tephra.TxConstants;
+import org.apache.twill.internal.utils.Networks;
 
 
 /**
@@ -69,6 +72,7 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
      * because we want to control it's execution ourselves
      */
     public static final long DEFAULT_INDEX_REBUILD_TASK_INITIAL_DELAY = Long.MAX_VALUE;
+    public static final int DEFAULT_TXN_TIMEOUT_SECONDS = 30;
 
     
     /**
@@ -117,7 +121,16 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
                 .setHConnectionPoolMaxSize(DEFAULT_HCONNECTION_POOL_MAX_SIZE)
                 .setMaxThreadsPerHTable(DEFAULT_HTABLE_MAX_THREADS)
                 .setDefaultIndexPopulationWaitTime(DEFAULT_INDEX_POPULATION_WAIT_TIME)
-                .setIndexRebuildTaskInitialDelay(DEFAULT_INDEX_REBUILD_TASK_INITIAL_DELAY);
+                .setIndexRebuildTaskInitialDelay(DEFAULT_INDEX_REBUILD_TASK_INITIAL_DELAY)
+                // setup default configs for Tephra
+                .set(TxConstants.Manager.CFG_DO_PERSIST, false)
+                .set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times")
+                .set(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1)
+                .set(TxConstants.Service.CFG_DATA_TX_BIND_PORT, Networks.getRandomPort())
+                .set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, Files.createTempDir().getAbsolutePath())
+                .set(TxConstants.Manager.CFG_TX_TIMEOUT, DEFAULT_TXN_TIMEOUT_SECONDS)
+                .set(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, 5L)
+                ;
     }
     
     public QueryServicesTestImpl(ReadOnlyProps defaultProps, ReadOnlyProps overrideProps) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index 1ec07b6..a06fd69 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -780,7 +780,7 @@ public class TestUtil {
         ConnectionQueryServices services = conn.unwrap(PhoenixConnection.class).getQueryServices();
         MutationState mutationState = pconn.getMutationState();
         if (table.isTransactional()) {
-            mutationState.startTransaction();
+            mutationState.startTransaction(table.getTransactionProvider());
         }
         try (HTableInterface htable = mutationState.getHTable(table)) {
             byte[] markerRowKey = Bytes.toBytes("TO_DELETE");

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-protocol/src/main/PTable.proto
----------------------------------------------------------------------
diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto
index ba9e0b4..16381dd 100644
--- a/phoenix-protocol/src/main/PTable.proto
+++ b/phoenix-protocol/src/main/PTable.proto
@@ -100,6 +100,7 @@ message PTable {
   optional bytes encodingScheme = 35;
   repeated EncodedCQCounter encodedCQCounters = 36;
   optional bool useStatsForParallelization = 37;
+  optional int32 transactionProvider = 38;
 }
 
 message EncodedCQCounter {


[18/50] [abbrv] phoenix git commit: PHOENIX-4579 Add a config to conditionally create Phoenix meta tables on first client connection (Chinmay Kulkarni)

Posted by td...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 914f62a..8b328d3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -170,6 +170,7 @@ import org.apache.phoenix.exception.RetriableUpgradeException;
 import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.exception.SQLExceptionInfo;
 import org.apache.phoenix.exception.UpgradeInProgressException;
+import org.apache.phoenix.exception.UpgradeRequiredException;
 import org.apache.phoenix.exception.UpgradeNotRequiredException;
 import org.apache.phoenix.execute.MutationState;
 import org.apache.phoenix.hbase.index.IndexRegionSplitPolicy;
@@ -1041,23 +1042,69 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
 
     /**
      *
-     * @param tableName
+     * @param physicalTableName
+     * @param tableType
+     * @param props
+     * @param families
      * @param splits
-     * @param modifyExistingMetaData TODO
+     * @param modifyExistingMetaData
+     * @param isNamespaceMapped
+     * @param isDoNotUpgradePropSet
      * @return true if table was created and false if it already exists
      * @throws SQLException
      */
     private HTableDescriptor ensureTableCreated(byte[] physicalTableName, PTableType tableType, Map<String, Object> props,
             List<Pair<byte[], Map<String, Object>>> families, byte[][] splits, boolean modifyExistingMetaData,
-            boolean isNamespaceMapped) throws SQLException {
+            boolean isNamespaceMapped, boolean isDoNotUpgradePropSet) throws SQLException {
         SQLException sqlE = null;
         HTableDescriptor existingDesc = null;
         boolean isMetaTable = SchemaUtil.isMetaTable(physicalTableName);
         boolean tableExist = true;
         try (HBaseAdmin admin = getAdmin()) {
             final String quorum = ZKConfig.getZKQuorumServersString(config);
-            final String znode = this.props.get(HConstants.ZOOKEEPER_ZNODE_PARENT);
+            final String znode = this.getProps().get(HConstants.ZOOKEEPER_ZNODE_PARENT);
             logger.debug("Found quorum: " + quorum + ":" + znode);
+
+            if (isMetaTable) {
+                if(SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, this.getProps())) {
+                    try {
+                        // SYSTEM namespace needs to be created via HBase APIs because "CREATE SCHEMA" statement tries to write
+                        // its metadata in SYSTEM:CATALOG table. Without SYSTEM namespace, SYSTEM:CATALOG table cannot be created
+                        ensureNamespaceCreated(QueryConstants.SYSTEM_SCHEMA_NAME);
+                    } catch (PhoenixIOException e) {
+                        // We could either:
+                        // 1) Not access the NS descriptor. The NS may or may not exist at this point
+                        // 2) We could not create the NS
+                        // Regardless of the case 1 or 2, if we eventually try to migrate SYSTEM tables to the SYSTEM
+                        // namespace using the {@link ensureSystemTablesMigratedToSystemNamespace(ReadOnlyProps)} method,
+                        // if the NS does not exist, we will error as expected, or
+                        // if the NS does exist and tables are already mapped, the check will exit gracefully
+                    }
+                    if (admin.tableExists(SchemaUtil.getPhysicalTableName(SYSTEM_CATALOG_NAME_BYTES, false))) {
+                        // SYSTEM.CATALOG exists, so at this point, we have 3 cases:
+                        // 1) If server-side namespace mapping is disabled, throw Inconsistent namespace mapping exception
+                        // 2) If server-side namespace mapping is enabled and SYSCAT needs to be upgraded, upgrade SYSCAT
+                        //    and also migrate SYSTEM tables to the SYSTEM namespace
+                        // 3. If server-side namespace mapping is enabled and SYSCAT doesn't need to be upgraded, we still
+                        //    need to migrate SYSTEM tables to the SYSTEM namespace using the
+                        //    {@link ensureSystemTablesMigratedToSystemNamespace(ReadOnlyProps)} method (as part of
+                        //    {@link upgradeSystemTables(String, Properties)})
+                        checkClientServerCompatibility(SYSTEM_CATALOG_NAME_BYTES);
+                        // Thrown so we can force an upgrade which will just migrate SYSTEM tables to the SYSTEM namespace
+                        throw new UpgradeRequiredException(MIN_SYSTEM_TABLE_TIMESTAMP);
+                    }
+                } else if (admin.tableExists(SchemaUtil.getPhysicalTableName(SYSTEM_CATALOG_NAME_BYTES, true))) {
+                    // If SYSTEM:CATALOG exists, but client-side namespace mapping for SYSTEM tables is disabled, throw an exception
+                    throw new SQLExceptionInfo.Builder(
+                      SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES)
+                      .setMessage("Cannot initiate connection as "
+                        + SchemaUtil.getPhysicalTableName(SYSTEM_CATALOG_NAME_BYTES, true)
+                        + " is found but client does not have "
+                        + IS_NAMESPACE_MAPPING_ENABLED + " enabled")
+                      .build().buildException();
+                }
+            }
+
             try {
                 existingDesc = admin.getTableDescriptor(physicalTableName);
             } catch (org.apache.hadoop.hbase.TableNotFoundException e) {
@@ -1075,6 +1122,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                     splits, isNamespaceMapped);
 
             if (!tableExist) {
+                if (isMetaTable && !isUpgradeRequired() && (!isAutoUpgradeEnabled || isDoNotUpgradePropSet)) {
+                    // Disallow creating the SYSTEM.CATALOG or SYSTEM:CATALOG HBase table
+                    throw new UpgradeRequiredException();
+                }
                 if (newDesc.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_BYTES) != null && Boolean.TRUE.equals(
                         PBoolean.INSTANCE.toObject(newDesc.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_BYTES)))) {
                     newDesc.setValue(HTableDescriptor.SPLIT_POLICY, IndexRegionSplitPolicy.class.getName());
@@ -1092,9 +1143,12 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                 } catch (TableExistsException e) {
                     // We can ignore this, as it just means that another client beat us
                     // to creating the HBase metadata.
+                    if (isMetaTable && !isUpgradeRequired()) {
+                        checkClientServerCompatibility(SchemaUtil.getPhysicalName(SYSTEM_CATALOG_NAME_BYTES, this.getProps()).getName());
+                    }
                     return null;
                 }
-                if (isMetaTable) {
+                if (isMetaTable && !isUpgradeRequired()) {
                     checkClientServerCompatibility(SchemaUtil.getPhysicalName(SYSTEM_CATALOG_NAME_BYTES, this.getProps()).getName());
                     /*
                      * Now we modify the table to add the split policy, since we know that the client and
@@ -1106,7 +1160,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                 }
                 return null;
             } else {
-                if (isMetaTable) {
+                if (isMetaTable && !isUpgradeRequired()) {
                     checkClientServerCompatibility(SchemaUtil.getPhysicalName(SYSTEM_CATALOG_NAME_BYTES, this.getProps()).getName());
                 } else {
                     for(Pair<byte[],Map<String,Object>> family: families) {
@@ -1120,7 +1174,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                     }
                 }
 
-
                 if (!modifyExistingMetaData) {
                     return existingDesc; // Caller already knows that no metadata was changed
                 }
@@ -1143,7 +1196,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                     return null; // Indicate that no metadata was changed
                 }
 
-                modifyTable(physicalTableName, newDesc, true);
+                // Do not call modifyTable for SYSTEM tables
+                if (tableType != PTableType.SYSTEM) {
+                    modifyTable(physicalTableName, newDesc, true);
+                }
                 return newDesc;
             }
 
@@ -1198,6 +1254,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         boolean isIncompatible = false;
         int minHBaseVersion = Integer.MAX_VALUE;
         boolean isTableNamespaceMappingEnabled = false;
+        long systemCatalogTimestamp = Long.MAX_VALUE;
         HTableInterface ht = null;
         try {
             List<HRegionLocation> locations = this
@@ -1214,36 +1271,44 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
             }
 
             ht = this.getTable(metaTable);
-            final Map<byte[], Long> results =
-                    ht.coprocessorService(MetaDataService.class, null, null, new Batch.Call<MetaDataService,Long>() {
+            final Map<byte[], GetVersionResponse> results =
+                    ht.coprocessorService(MetaDataService.class, null, null, new Batch.Call<MetaDataService,GetVersionResponse>() {
                         @Override
-                        public Long call(MetaDataService instance) throws IOException {
+                        public GetVersionResponse call(MetaDataService instance) throws IOException {
                             ServerRpcController controller = new ServerRpcController();
                             BlockingRpcCallback<GetVersionResponse> rpcCallback =
-                                    new BlockingRpcCallback<GetVersionResponse>();
+                                    new BlockingRpcCallback<>();
                             GetVersionRequest.Builder builder = GetVersionRequest.newBuilder();
                             builder.setClientVersion(VersionUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER));
                             instance.getVersion(controller, builder.build(), rpcCallback);
                             if(controller.getFailedOn() != null) {
                                 throw controller.getFailedOn();
                             }
-                            return rpcCallback.get().getVersion();
+                            return rpcCallback.get();
                         }
                     });
-            for (Map.Entry<byte[],Long> result : results.entrySet()) {
+            for (Map.Entry<byte[],GetVersionResponse> result : results.entrySet()) {
                 // This is the "phoenix.jar" is in-place, but server is out-of-sync with client case.
-                long version = result.getValue();
-                isTableNamespaceMappingEnabled |= MetaDataUtil.decodeTableNamespaceMappingEnabled(version);
+                GetVersionResponse versionResponse = result.getValue();
+                long serverJarVersion = versionResponse.getVersion();
+                isTableNamespaceMappingEnabled |= MetaDataUtil.decodeTableNamespaceMappingEnabled(serverJarVersion);
 
-                if (!isCompatible(result.getValue())) {
+                if (!isCompatible(serverJarVersion)) {
                     isIncompatible = true;
                     HRegionLocation name = regionMap.get(result.getKey());
                     buf.append(name);
                     buf.append(';');
                 }
-                hasIndexWALCodec &= hasIndexWALCodec(result.getValue());
-                if (minHBaseVersion > MetaDataUtil.decodeHBaseVersion(result.getValue())) {
-                    minHBaseVersion = MetaDataUtil.decodeHBaseVersion(result.getValue());
+                hasIndexWALCodec &= hasIndexWALCodec(serverJarVersion);
+                if (minHBaseVersion > MetaDataUtil.decodeHBaseVersion(serverJarVersion)) {
+                    minHBaseVersion = MetaDataUtil.decodeHBaseVersion(serverJarVersion);
+                }
+                // In case this is the first time connecting to this cluster, the system catalog table does not have an
+                // entry for itself yet, so we cannot get the timestamp and this will not be returned from the
+                // GetVersionResponse message object
+                if (versionResponse.hasSystemCatalogTimestamp()) {
+                    systemCatalogTimestamp = systemCatalogTimestamp < versionResponse.getSystemCatalogTimestamp() ?
+                      systemCatalogTimestamp: versionResponse.getSystemCatalogTimestamp();
                 }
             }
             if (isTableNamespaceMappingEnabled != SchemaUtil.isNamespaceMappingEnabled(PTableType.TABLE,
@@ -1274,6 +1339,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
             buf.setLength(buf.length()-1);
             throw new SQLExceptionInfo.Builder(SQLExceptionCode.OUTDATED_JARS).setMessage(buf.toString()).build().buildException();
         }
+        if (systemCatalogTimestamp < MIN_SYSTEM_TABLE_TIMESTAMP) {
+            throw new UpgradeRequiredException(systemCatalogTimestamp);
+        }
     }
 
     /**
@@ -1334,14 +1402,14 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
 
         tableProps.put(MetaDataUtil.IS_VIEW_INDEX_TABLE_PROP_NAME, TRUE_BYTES_AS_STRING);
         HTableDescriptor desc = ensureTableCreated(physicalIndexName, PTableType.TABLE, tableProps, families, splits,
-                false, isNamespaceMapped);
+                false, isNamespaceMapped, false);
         if (desc != null) {
             if (!Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(desc.getValue(MetaDataUtil.IS_VIEW_INDEX_TABLE_PROP_BYTES)))) {
                 String fullTableName = Bytes.toString(physicalIndexName);
                 throw new TableAlreadyExistsException(
-                        "Unable to create shared physical table for indexes on views.",
-                        SchemaUtil.getSchemaNameFromFullName(fullTableName),
-                        SchemaUtil.getTableNameFromFullName(fullTableName));
+                  SchemaUtil.getSchemaNameFromFullName(fullTableName),
+                  SchemaUtil.getTableNameFromFullName(fullTableName),
+                  "Unable to create shared physical table for indexes on views.");
             }
         }
     }
@@ -1409,8 +1477,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     @Override
     public MetaDataMutationResult createTable(final List<Mutation> tableMetaData, final byte[] physicalTableName,
             PTableType tableType, Map<String, Object> tableProps,
-            final List<Pair<byte[], Map<String, Object>>> families, byte[][] splits, boolean isNamespaceMapped, final boolean allocateIndexId)
-                    throws SQLException {
+            final List<Pair<byte[], Map<String, Object>>> families, byte[][] splits, boolean isNamespaceMapped,
+      final boolean allocateIndexId, final boolean isDoNotUpgradePropSet) throws SQLException {
         byte[][] rowKeyMetadata = new byte[3][];
         Mutation m = MetaDataUtil.getPutOnlyTableHeaderRow(tableMetaData);
         byte[] key = m.getRow();
@@ -1430,7 +1498,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         if ((tableType == PTableType.VIEW && physicalTableName != null) || (tableType != PTableType.VIEW && (physicalTableName == null || localIndexTable))) {
             // For views this will ensure that metadata already exists
             // For tables and indexes, this will create the metadata if it doesn't already exist
-            ensureTableCreated(tableName, tableType, tableProps, families, splits, true, isNamespaceMapped);
+            ensureTableCreated(tableName, tableType, tableProps, families, splits, true, isNamespaceMapped, isDoNotUpgradePropSet);
         }
         ImmutableBytesWritable ptr = new ImmutableBytesWritable();
         if (tableType == PTableType.INDEX) { // Index on view
@@ -2436,30 +2504,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                             openConnection();
                             hConnectionEstablished = true;
                             boolean isDoNotUpgradePropSet = UpgradeUtil.isNoUpgradeSet(props);
-                            try (HBaseAdmin admin = getAdmin()) {
-                                boolean mappedSystemCatalogExists = admin
-                                        .tableExists(SchemaUtil.getPhysicalTableName(SYSTEM_CATALOG_NAME_BYTES, true));
-                                if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM,
-                                        ConnectionQueryServicesImpl.this.getProps())) {
-                                    if (admin.tableExists(SYSTEM_CATALOG_NAME_BYTES)) {
-                                        //check if the server is already updated and have namespace config properly set. 
-                                        checkClientServerCompatibility(SYSTEM_CATALOG_NAME_BYTES);
-                                    }
-
-                                    // If SYSTEM tables exist, they are migrated to HBase SYSTEM namespace
-                                    // If they don't exist, this method will create HBase SYSTEM namespace and return
-                                    ensureSystemTablesMigratedToSystemNamespace(ConnectionQueryServicesImpl.this.getProps());
-                                } else if (mappedSystemCatalogExists) {
-                                    throw new SQLExceptionInfo.Builder(
-                                            SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES)
-                                            .setMessage("Cannot initiate connection as "
-                                                    + SchemaUtil.getPhysicalTableName(
-                                                    SYSTEM_CATALOG_NAME_BYTES, true)
-                                                    + " is found but client does not have "
-                                                    + IS_NAMESPACE_MAPPING_ENABLED + " enabled")
-                                            .build().buildException();
-                                }
-                            }
                             Properties scnProps = PropertiesUtil.deepCopy(props);
                             scnProps.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
                                     Long.toString(getSystemTableVersion()));
@@ -2508,38 +2552,37 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                                         initializationException = e;
                                     }
                                     return null;
+                                } catch (UpgradeRequiredException e) {
+                                    // This will occur in 3 cases:
+                                    // 1. SYSTEM.CATALOG does not exist and we don't want to allow the user to create it i.e.
+                                    //    !isAutoUpgradeEnabled or isDoNotUpgradePropSet is set
+                                    // 2. SYSTEM.CATALOG exists and its timestamp < MIN_SYSTEM_TABLE_TIMESTAMP
+                                    // 3. SYSTEM.CATALOG exists, but client and server-side namespace mapping is enabled so
+                                    //    we need to migrate SYSTEM tables to the SYSTEM namespace
+                                    setUpgradeRequired();
                                 }
 
-                                // HBase Namespace SYSTEM is created by {@link ensureSystemTablesMigratedToSystemNamespace(ReadOnlyProps)} method
-                                // This statement will create its entry in SYSCAT table, so that GRANT/REVOKE commands can work
-                                // with SYSTEM Namespace. (See PHOENIX-4227 https://issues.apache.org/jira/browse/PHOENIX-4227)
-                                if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM,
-                                        ConnectionQueryServicesImpl.this.getProps())) {
-                                    try {
-                                        metaConnection.createStatement().execute("CREATE SCHEMA IF NOT EXISTS "
-                                                + PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA);
-                                    } catch (NewerSchemaAlreadyExistsException e) {
-                                        // Older clients with appropriate perms may try getting a new connection
-                                        // This results in NewerSchemaAlreadyExistsException, so we can safely ignore it here
-                                    } catch (PhoenixIOException e) {
-                                        if (!Iterables.isEmpty(Iterables.filter(Throwables.getCausalChain(e), AccessDeniedException.class))) {
-                                            // Ignore ADE
-                                        } else {
-                                            throw e;
-                                        }
-                                    }
-                                }
                                 if (!ConnectionQueryServicesImpl.this.upgradeRequired.get()) {
                                     createOtherSystemTables(metaConnection, hBaseAdmin);
+                                    // In case namespace mapping is enabled and system table to system namespace mapping is also enabled,
+                                    // create an entry for the SYSTEM namespace in the SYSCAT table, so that GRANT/REVOKE commands can work
+                                    // with SYSTEM Namespace
+                                    createSchemaIfNotExistsSystemNSMappingEnabled(metaConnection);
                                 } else if (isAutoUpgradeEnabled && !isDoNotUpgradePropSet) {
+                                    // Upgrade is required and we are allowed to automatically upgrade
                                     upgradeSystemTables(url, props);
+                                } else {
+                                    // We expect the user to manually run the "EXECUTE UPGRADE" command first.
+                                    // This exception will get caught below as a RetriableUpgradeException
+                                    throw new UpgradeRequiredException();
                                 }
                             }
                             scheduleRenewLeaseTasks();
                             success = true;
                         } catch (RetriableUpgradeException e) {
-                            // Don't set it as initializationException because otherwise the clien't won't be able
-                            // to retry establishing connection.
+                            // Set success to true and don't set the exception as an initializationException,
+                            // because otherwise the client won't be able to retry establishing the connection.
+                            success = true;
                             throw e;
                         } catch (Exception e) {
                             if (e instanceof SQLException) {
@@ -2580,7 +2623,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         }
     }
 
-    void createSysMutexTableIfNotExists(HBaseAdmin admin, ReadOnlyProps props) throws IOException, SQLException {
+    void createSysMutexTableIfNotExists(HBaseAdmin admin) throws IOException, SQLException {
         try {
             if(admin.tableExists(PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME) || admin.tableExists(TableName.valueOf(
                     PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME,PhoenixDatabaseMetaData.SYSTEM_MUTEX_TABLE_NAME))) {
@@ -2588,7 +2631,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                 return;
             }
             final TableName mutexTableName = SchemaUtil.getPhysicalTableName(
-                PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME, props);
+                PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME, this.getProps());
             HTableDescriptor tableDesc = new HTableDescriptor(mutexTableName);
             HColumnDescriptor columnDesc = new HColumnDescriptor(
                     PhoenixDatabaseMetaData.SYSTEM_MUTEX_FAMILY_NAME_BYTES);
@@ -2640,7 +2683,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         } catch (TableAlreadyExistsException ignore) {}
         // Catch the IOException to log the error message and then bubble it up for the client to retry.
         try {
-            createSysMutexTableIfNotExists(hbaseAdmin, ConnectionQueryServicesImpl.this.getProps());
+            createSysMutexTableIfNotExists(hbaseAdmin);
         } catch (IOException exception) {
             logger.error("Failed to created SYSMUTEX table. Upgrade or migration is not possible without it. Please retry.");
             throw exception;
@@ -2648,6 +2691,265 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     }
 
     /**
+     * Create an entry for the SYSTEM namespace in the SYSCAT table in case namespace mapping is enabled and system table
+     * to system namespace mapping is also enabled. If not enabled, this method returns immediately without doing anything
+     * @param metaConnection
+     * @throws SQLException
+     */
+    private void createSchemaIfNotExistsSystemNSMappingEnabled(PhoenixConnection metaConnection) throws SQLException {
+        // HBase Namespace SYSTEM is assumed to be already created inside {@link ensureTableCreated(byte[], PTableType,
+        // Map<String, Object>, List<Pair<byte[], Map<String, Object>>>, byte[][], boolean, boolean, boolean)}.
+        // This statement will create an entry for the SYSTEM namespace in the SYSCAT table, so that GRANT/REVOKE
+        // commands can work with SYSTEM Namespace. (See PHOENIX-4227 https://issues.apache.org/jira/browse/PHOENIX-4227)
+        if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM,
+          ConnectionQueryServicesImpl.this.getProps())) {
+            try {
+                metaConnection.createStatement().execute("CREATE SCHEMA IF NOT EXISTS "
+                  + PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA);
+            } catch (NewerSchemaAlreadyExistsException e) {
+                // Older clients with appropriate perms may try getting a new connection
+                // This results in NewerSchemaAlreadyExistsException, so we can safely ignore it here
+            } catch (PhoenixIOException e) {
+                if (!Iterables.isEmpty(Iterables.filter(Throwables.getCausalChain(e), AccessDeniedException.class))) {
+                    // Ignore ADE
+                } else {
+                    throw e;
+                }
+            }
+        }
+    }
+
+    /**
+     * Upgrade the SYSCAT schema if required
+     * @param metaConnection
+     * @param currentServerSideTableTimeStamp
+     * @return Phoenix connection object
+     * @throws SQLException
+     * @throws IOException
+     * @throws TimeoutException
+     * @throws InterruptedException
+     */
+    // Available for testing
+    protected PhoenixConnection upgradeSystemCatalogIfRequired(PhoenixConnection metaConnection,
+      long currentServerSideTableTimeStamp) throws SQLException, IOException, TimeoutException, InterruptedException {
+        String columnsToAdd = "";
+        // This will occur if we have an older SYSTEM.CATALOG and we need to update it to
+        // include any new columns we've added.
+        if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0) {
+            // We know that we always need to add the STORE_NULLS column for 4.3 release
+            columnsToAdd = addColumn(columnsToAdd, PhoenixDatabaseMetaData.STORE_NULLS
+              + " " + PBoolean.INSTANCE.getSqlTypeName());
+            try (HBaseAdmin admin = getAdmin()) {
+                HTableDescriptor[] localIndexTables = admin
+                  .listTables(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX + ".*");
+                for (HTableDescriptor table : localIndexTables) {
+                    if (table.getValue(MetaDataUtil.PARENT_TABLE_KEY) == null
+                      && table.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_NAME) != null) {
+                        table.setValue(MetaDataUtil.PARENT_TABLE_KEY,
+                          MetaDataUtil.getLocalIndexUserTableName(table.getNameAsString()));
+                        // Explicitly disable, modify and enable the table to ensure
+                        // co-location of data and index regions. If we just modify the
+                        // table descriptor when online schema change enabled may reopen
+                        // the region in same region server instead of following data region.
+                        admin.disableTable(table.getTableName());
+                        admin.modifyTable(table.getTableName(), table);
+                        admin.enableTable(table.getTableName());
+                    }
+                }
+            }
+        }
+
+        // If the server side schema is before MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0 then
+        // we need to add INDEX_TYPE and INDEX_DISABLE_TIMESTAMP columns too.
+        // TODO: Once https://issues.apache.org/jira/browse/PHOENIX-1614 is fixed,
+        // we should just have a ALTER TABLE ADD IF NOT EXISTS statement with all
+        // the column names that have been added to SYSTEM.CATALOG since 4.0.
+        if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0) {
+            columnsToAdd = addColumn(columnsToAdd, PhoenixDatabaseMetaData.INDEX_TYPE + " "
+              + PUnsignedTinyint.INSTANCE.getSqlTypeName() + ", "
+              + PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP + " "
+              + PLong.INSTANCE.getSqlTypeName());
+        }
+
+        // If we have some new columns from 4.1-4.3 to add, add them now.
+        if (!columnsToAdd.isEmpty()) {
+            // Ugh..need to assign to another local variable to keep eclipse happy.
+            PhoenixConnection newMetaConnection = addColumnsIfNotExists(metaConnection,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0, columnsToAdd);
+            metaConnection = newMetaConnection;
+        }
+
+        if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0) {
+            columnsToAdd = PhoenixDatabaseMetaData.BASE_COLUMN_COUNT + " "
+              + PInteger.INSTANCE.getSqlTypeName();
+            try {
+                metaConnection = addColumn(metaConnection,
+                  PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+                  MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0, columnsToAdd,
+                  false);
+                upgradeTo4_5_0(metaConnection);
+            } catch (ColumnAlreadyExistsException ignored) {
+                /*
+                 * Upgrade to 4.5 is a slightly special case. We use the fact that the
+                 * column BASE_COLUMN_COUNT is already part of the meta-data schema as the
+                 * signal that the server side upgrade has finished or is in progress.
+                 */
+                logger.debug("No need to run 4.5 upgrade");
+            }
+            Properties p = PropertiesUtil.deepCopy(metaConnection.getClientInfo());
+            p.remove(PhoenixRuntime.CURRENT_SCN_ATTRIB);
+            p.remove(PhoenixRuntime.TENANT_ID_ATTRIB);
+            PhoenixConnection conn = new PhoenixConnection(
+              ConnectionQueryServicesImpl.this, metaConnection.getURL(), p,
+              metaConnection.getMetaDataCache());
+            try {
+                List<String> tablesNeedingUpgrade = UpgradeUtil
+                  .getPhysicalTablesWithDescRowKey(conn);
+                if (!tablesNeedingUpgrade.isEmpty()) {
+                    logger.warn("The following tables require upgrade due to a bug causing the row key to be incorrect for descending columns and ascending BINARY columns (PHOENIX-2067 and PHOENIX-2120):\n"
+                      + Joiner.on(' ').join(tablesNeedingUpgrade)
+                      + "\nTo upgrade issue the \"bin/psql.py -u\" command.");
+                }
+                List<String> unsupportedTables = UpgradeUtil
+                  .getPhysicalTablesWithDescVarbinaryRowKey(conn);
+                if (!unsupportedTables.isEmpty()) {
+                    logger.warn("The following tables use an unsupported VARBINARY DESC construct and need to be changed:\n"
+                      + Joiner.on(' ').join(unsupportedTables));
+                }
+            } catch (Exception ex) {
+                logger.error(
+                  "Unable to determine tables requiring upgrade due to PHOENIX-2067",
+                  ex);
+            } finally {
+                conn.close();
+            }
+        }
+        // Add these columns one at a time, each with different timestamps so that if folks
+        // have
+        // run the upgrade code already for a snapshot, we'll still enter this block (and do
+        // the
+        // parts we haven't yet done).
+        if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0) {
+            columnsToAdd = PhoenixDatabaseMetaData.IS_ROW_TIMESTAMP + " "
+              + PBoolean.INSTANCE.getSqlTypeName();
+            metaConnection = addColumnsIfNotExists(metaConnection,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, columnsToAdd);
+        }
+        if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) {
+            // Drop old stats table so that new stats table is created
+            metaConnection = dropStatsTable(metaConnection,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 4);
+            metaConnection = addColumnsIfNotExists(
+              metaConnection,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 3,
+              PhoenixDatabaseMetaData.TRANSACTIONAL + " "
+                + PBoolean.INSTANCE.getSqlTypeName());
+            metaConnection = addColumnsIfNotExists(
+              metaConnection,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 2,
+              PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " "
+                + PLong.INSTANCE.getSqlTypeName());
+            metaConnection = setImmutableTableIndexesImmutable(metaConnection,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1);
+            metaConnection = updateSystemCatalogTimestamp(metaConnection,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
+            ConnectionQueryServicesImpl.this.removeTable(null,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
+            clearCache();
+        }
+
+        if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0) {
+            metaConnection = addColumnsIfNotExists(
+              metaConnection,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 - 2,
+              PhoenixDatabaseMetaData.IS_NAMESPACE_MAPPED + " "
+                + PBoolean.INSTANCE.getSqlTypeName());
+            metaConnection = addColumnsIfNotExists(
+              metaConnection,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 - 1,
+              PhoenixDatabaseMetaData.AUTO_PARTITION_SEQ + " "
+                + PVarchar.INSTANCE.getSqlTypeName());
+            metaConnection = addColumnsIfNotExists(
+              metaConnection,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0,
+              PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA + " "
+                + PBoolean.INSTANCE.getSqlTypeName());
+            metaConnection = UpgradeUtil.disableViewIndexes(metaConnection);
+            if (getProps().getBoolean(QueryServices.LOCAL_INDEX_CLIENT_UPGRADE_ATTRIB,
+              QueryServicesOptions.DEFAULT_LOCAL_INDEX_CLIENT_UPGRADE)) {
+                metaConnection = UpgradeUtil.upgradeLocalIndexes(metaConnection);
+            }
+            ConnectionQueryServicesImpl.this.removeTable(null,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0);
+            clearCache();
+        }
+        if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0) {
+            metaConnection = addColumnsIfNotExists(
+              metaConnection,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0,
+              PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + " "
+                + PLong.INSTANCE.getSqlTypeName());
+            ConnectionQueryServicesImpl.this.removeTable(null,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0);
+            clearCache();
+        }
+        if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0) {
+            metaConnection = addColumnQualifierColumn(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 3);
+            metaConnection = addColumnsIfNotExists(
+              metaConnection,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 2,
+              PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME + " "
+                + PTinyint.INSTANCE.getSqlTypeName());
+            metaConnection = addColumnsIfNotExists(
+              metaConnection,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 1,
+              PhoenixDatabaseMetaData.ENCODING_SCHEME + " "
+                + PTinyint.INSTANCE.getSqlTypeName());
+            metaConnection = addColumnsIfNotExists(
+              metaConnection,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0,
+              PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER + " "
+                + PInteger.INSTANCE.getSqlTypeName());
+            ConnectionQueryServicesImpl.this.removeTable(null,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0);
+            clearCache();
+        }
+        if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0) {
+            metaConnection = addColumnsIfNotExists(
+              metaConnection,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0,
+              PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION + " "
+                + PBoolean.INSTANCE.getSqlTypeName());
+            addParentToChildLinks(metaConnection);
+        }
+        if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0) {
+            metaConnection = addColumnsIfNotExists(
+              metaConnection,
+              PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+              MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0,
+              PhoenixDatabaseMetaData.TRANSACTION_PROVIDER + " "
+                + PTinyint.INSTANCE.getSqlTypeName());
+        }
+        return metaConnection;
+    }
+
+    /**
      * There is no other locking needed here since only one connection (on the same or different JVM) will be able to
      * acquire the upgrade mutex via {@link #acquireUpgradeMutex(long, byte[])}.
      */
@@ -2674,244 +2976,54 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
             metaConnection = new PhoenixConnection(ConnectionQueryServicesImpl.this, globalUrl,
                     scnProps, newEmptyMetaData());
             metaConnection.setRunningUpgrade(true);
+            // Always try to create SYSTEM.MUTEX table since we need it to acquire the upgrade mutex.
+            // Upgrade or migration is not possible without the upgrade mutex
+            try (HBaseAdmin admin = getAdmin()) {
+                createSysMutexTableIfNotExists(admin);
+            }
             try {
                 metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_TABLE_METADATA);
             } catch (NewerTableAlreadyExistsException ignore) {
                 // Ignore, as this will happen if the SYSTEM.CATALOG already exists at this fixed
                 // timestamp. A TableAlreadyExistsException is not thrown, since the table only exists
                 // *after* this fixed timestamp.
+            } catch (UpgradeRequiredException e) {
+                // This is thrown while trying to create SYSTEM:CATALOG to indicate that we must migrate SYSTEM tables
+                // to the SYSTEM namespace and/or upgrade SYSCAT if required
+                sysCatalogTableName = SchemaUtil.getPhysicalName(SYSTEM_CATALOG_NAME_BYTES, this.getProps()).getNameAsString();
+                if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, ConnectionQueryServicesImpl.this.getProps())) {
+                    // Try acquiring a lock in SYSMUTEX table before migrating the tables since it involves disabling the table.
+                    if (acquiredMutexLock = acquireUpgradeMutex(MetaDataProtocol.MIN_SYSTEM_TABLE_MIGRATION_TIMESTAMP, mutexRowKey)) {
+                        logger.debug("Acquired lock in SYSMUTEX table for migrating SYSTEM tables to SYSTEM namespace "
+                          + "and/or upgrading " + sysCatalogTableName);
+                    }
+                    // We will not reach here if we fail to acquire the lock, since it throws UpgradeInProgressException
+
+                    // If SYSTEM tables exist, they are migrated to HBase SYSTEM namespace
+                    // If they don't exist or they're already migrated, this method will return immediately
+                    ensureSystemTablesMigratedToSystemNamespace();
+                    logger.debug("Migrated SYSTEM tables to SYSTEM namespace");
+                    metaConnection = upgradeSystemCatalogIfRequired(metaConnection, e.getSystemCatalogTimeStamp());
+                }
             } catch (TableAlreadyExistsException e) {
                 long currentServerSideTableTimeStamp = e.getTable().getTimeStamp();
                 sysCatalogTableName = e.getTable().getPhysicalName().getString();
                 if (currentServerSideTableTimeStamp < MIN_SYSTEM_TABLE_TIMESTAMP) {
-                    if (currentServerSideTableTimeStamp <= MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0) {
-                        try (HBaseAdmin admin = getAdmin()) {
-                            createSysMutexTableIfNotExists(admin, this.getProps());
-                        }
-                    }
+                    // Try acquiring a lock in SYSMUTEX table before upgrading SYSCAT. If we cannot acquire the lock,
+                    // it means some old client is either migrating SYSTEM tables or trying to upgrade the schema of
+                    // SYSCAT table and hence it should not be interrupted
                     if (acquiredMutexLock = acquireUpgradeMutex(currentServerSideTableTimeStamp, mutexRowKey)) {
+                        logger.debug("Acquired lock in SYSMUTEX table for upgrading " + sysCatalogTableName);
                         snapshotName = getSysCatalogSnapshotName(currentServerSideTableTimeStamp);
                         createSnapshot(snapshotName, sysCatalogTableName);
                         snapshotCreated = true;
+                        logger.debug("Created snapshot for SYSCAT");
                     }
+                    // We will not reach here if we fail to acquire the lock, since it throws UpgradeInProgressException
                 }
-                String columnsToAdd = "";
-                // This will occur if we have an older SYSTEM.CATALOG and we need to update it to
-                // include any new columns we've added.
-                if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0) {
-                    // We know that we always need to add the STORE_NULLS column for 4.3 release
-                    columnsToAdd = addColumn(columnsToAdd, PhoenixDatabaseMetaData.STORE_NULLS
-                            + " " + PBoolean.INSTANCE.getSqlTypeName());
-                    try (HBaseAdmin admin = getAdmin()) {
-                        HTableDescriptor[] localIndexTables = admin
-                                .listTables(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX + ".*");
-                        for (HTableDescriptor table : localIndexTables) {
-                            if (table.getValue(MetaDataUtil.PARENT_TABLE_KEY) == null
-                                    && table.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_NAME) != null) {
-                                table.setValue(MetaDataUtil.PARENT_TABLE_KEY,
-                                        MetaDataUtil.getLocalIndexUserTableName(table.getNameAsString()));
-                                // Explicitly disable, modify and enable the table to ensure
-                                // co-location of data and index regions. If we just modify the
-                                // table descriptor when online schema change enabled may reopen 
-                                // the region in same region server instead of following data region.
-                                admin.disableTable(table.getTableName());
-                                admin.modifyTable(table.getTableName(), table);
-                                admin.enableTable(table.getTableName());
-                            }
-                        }
-                    }
-                }
-
-                // If the server side schema is before MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0 then
-                // we need to add INDEX_TYPE and INDEX_DISABLE_TIMESTAMP columns too.
-                // TODO: Once https://issues.apache.org/jira/browse/PHOENIX-1614 is fixed,
-                // we should just have a ALTER TABLE ADD IF NOT EXISTS statement with all
-                // the column names that have been added to SYSTEM.CATALOG since 4.0.
-                if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0) {
-                    columnsToAdd = addColumn(columnsToAdd, PhoenixDatabaseMetaData.INDEX_TYPE + " "
-                            + PUnsignedTinyint.INSTANCE.getSqlTypeName() + ", "
-                            + PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP + " "
-                            + PLong.INSTANCE.getSqlTypeName());
-                }
-
-                // If we have some new columns from 4.1-4.3 to add, add them now.
-                if (!columnsToAdd.isEmpty()) {
-                    // Ugh..need to assign to another local variable to keep eclipse happy.
-                    PhoenixConnection newMetaConnection = addColumnsIfNotExists(metaConnection,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0, columnsToAdd);
-                    metaConnection = newMetaConnection;
-                }
-
-                if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0) {
-                    columnsToAdd = PhoenixDatabaseMetaData.BASE_COLUMN_COUNT + " "
-                            + PInteger.INSTANCE.getSqlTypeName();
-                    try {
-                        metaConnection = addColumn(metaConnection,
-                                PhoenixDatabaseMetaData.SYSTEM_CATALOG,
-                                MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0, columnsToAdd,
-                                false);
-                        upgradeTo4_5_0(metaConnection);
-                    } catch (ColumnAlreadyExistsException ignored) {
-                        /*
-                         * Upgrade to 4.5 is a slightly special case. We use the fact that the
-                         * column BASE_COLUMN_COUNT is already part of the meta-data schema as the
-                         * signal that the server side upgrade has finished or is in progress.
-                         */
-                        logger.debug("No need to run 4.5 upgrade");
-                    }
-                    Properties p = PropertiesUtil.deepCopy(metaConnection.getClientInfo());
-                    p.remove(PhoenixRuntime.CURRENT_SCN_ATTRIB);
-                    p.remove(PhoenixRuntime.TENANT_ID_ATTRIB);
-                    PhoenixConnection conn = new PhoenixConnection(
-                            ConnectionQueryServicesImpl.this, metaConnection.getURL(), p,
-                            metaConnection.getMetaDataCache());
-                    try {
-                        List<String> tablesNeedingUpgrade = UpgradeUtil
-                                .getPhysicalTablesWithDescRowKey(conn);
-                        if (!tablesNeedingUpgrade.isEmpty()) {
-                            logger.warn("The following tables require upgrade due to a bug causing the row key to be incorrect for descending columns and ascending BINARY columns (PHOENIX-2067 and PHOENIX-2120):\n"
-                                    + Joiner.on(' ').join(tablesNeedingUpgrade)
-                                    + "\nTo upgrade issue the \"bin/psql.py -u\" command.");
-                        }
-                        List<String> unsupportedTables = UpgradeUtil
-                                .getPhysicalTablesWithDescVarbinaryRowKey(conn);
-                        if (!unsupportedTables.isEmpty()) {
-                            logger.warn("The following tables use an unsupported VARBINARY DESC construct and need to be changed:\n"
-                                    + Joiner.on(' ').join(unsupportedTables));
-                        }
-                    } catch (Exception ex) {
-                        logger.error(
-                                "Unable to determine tables requiring upgrade due to PHOENIX-2067",
-                                ex);
-                    } finally {
-                        conn.close();
-                    }
-                }
-                // Add these columns one at a time, each with different timestamps so that if folks
-                // have
-                // run the upgrade code already for a snapshot, we'll still enter this block (and do
-                // the
-                // parts we haven't yet done).
-                if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0) {
-                    columnsToAdd = PhoenixDatabaseMetaData.IS_ROW_TIMESTAMP + " "
-                            + PBoolean.INSTANCE.getSqlTypeName();
-                    metaConnection = addColumnsIfNotExists(metaConnection,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, columnsToAdd);
-                }
-                if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) {
-                    // Drop old stats table so that new stats table is created
-                    metaConnection = dropStatsTable(metaConnection,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 4);
-                    metaConnection = addColumnsIfNotExists(
-                            metaConnection,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 3,
-                            PhoenixDatabaseMetaData.TRANSACTIONAL + " "
-                                    + PBoolean.INSTANCE.getSqlTypeName());
-                    metaConnection = addColumnsIfNotExists(
-                            metaConnection,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 2,
-                            PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " "
-                                    + PLong.INSTANCE.getSqlTypeName());
-                    metaConnection = setImmutableTableIndexesImmutable(metaConnection,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1);
-                    metaConnection = updateSystemCatalogTimestamp(metaConnection,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
-                    ConnectionQueryServicesImpl.this.removeTable(null,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
-                    clearCache();
-                }
-
-                if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0) {
-                    metaConnection = addColumnsIfNotExists(
-                            metaConnection,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 - 2,
-                            PhoenixDatabaseMetaData.IS_NAMESPACE_MAPPED + " "
-                                    + PBoolean.INSTANCE.getSqlTypeName());
-                    metaConnection = addColumnsIfNotExists(
-                            metaConnection,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 - 1,
-                            PhoenixDatabaseMetaData.AUTO_PARTITION_SEQ + " "
-                                    + PVarchar.INSTANCE.getSqlTypeName());
-                    metaConnection = addColumnsIfNotExists(
-                            metaConnection,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0,
-                            PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA + " "
-                                    + PBoolean.INSTANCE.getSqlTypeName());
-                    metaConnection = UpgradeUtil.disableViewIndexes(metaConnection);
-                    if (getProps().getBoolean(QueryServices.LOCAL_INDEX_CLIENT_UPGRADE_ATTRIB,
-                            QueryServicesOptions.DEFAULT_LOCAL_INDEX_CLIENT_UPGRADE)) {
-                        metaConnection = UpgradeUtil.upgradeLocalIndexes(metaConnection);
-                    }
-                    ConnectionQueryServicesImpl.this.removeTable(null,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0);
-                    clearCache();
-                }
-                if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0) {
-                    metaConnection = addColumnsIfNotExists(
-                            metaConnection,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0,
-                            PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + " "
-                                    + PLong.INSTANCE.getSqlTypeName());
-                    ConnectionQueryServicesImpl.this.removeTable(null,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0);
-                    clearCache();
-                }
-                if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0) {
-                    metaConnection = addColumnQualifierColumn(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 3);
-                    metaConnection = addColumnsIfNotExists(
-                            metaConnection,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 2,
-                            PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME + " "
-                                    + PTinyint.INSTANCE.getSqlTypeName());
-                    metaConnection = addColumnsIfNotExists(
-                            metaConnection,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 1,
-                            PhoenixDatabaseMetaData.ENCODING_SCHEME + " "
-                                    + PTinyint.INSTANCE.getSqlTypeName());
-                    metaConnection = addColumnsIfNotExists(
-                            metaConnection,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0,
-                            PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER + " "
-                                    + PInteger.INSTANCE.getSqlTypeName());
-                    ConnectionQueryServicesImpl.this.removeTable(null,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
-                            MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0);
-                    clearCache();
-                }
-                if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0) {
-                    metaConnection = addColumnsIfNotExists(
-                        metaConnection,
-                        PhoenixDatabaseMetaData.SYSTEM_CATALOG,
-                        MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0,
-                        PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION + " "
-                                + PBoolean.INSTANCE.getSqlTypeName());
-                    addParentToChildLinks(metaConnection);
-                }
-                if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0) {
-                    metaConnection = addColumnsIfNotExists(
-                        metaConnection,
-                        PhoenixDatabaseMetaData.SYSTEM_CATALOG,
-                        MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0,
-                        PhoenixDatabaseMetaData.TRANSACTION_PROVIDER + " "
-                                + PTinyint.INSTANCE.getSqlTypeName());
-                }
+                metaConnection = upgradeSystemCatalogIfRequired(metaConnection, currentServerSideTableTimeStamp);
             }
 
-
             int nSaltBuckets = ConnectionQueryServicesImpl.this.props.getInt(
                     QueryServices.SEQUENCE_SALT_BUCKETS_ATTRIB,
                     QueryServicesOptions.DEFAULT_SEQUENCE_TABLE_SALT_BUCKETS);
@@ -2997,6 +3109,12 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
             try {
                 metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_LOG_METADATA);
             } catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {}
+
+            // In case namespace mapping is enabled and system table to system namespace mapping is also enabled,
+            // create an entry for the SYSTEM namespace in the SYSCAT table, so that GRANT/REVOKE commands can work
+            // with SYSTEM Namespace
+            createSchemaIfNotExistsSystemNSMappingEnabled(metaConnection);
+
             ConnectionQueryServicesImpl.this.upgradeRequired.set(false);
             success = true;
         } catch (UpgradeInProgressException | UpgradeNotRequiredException e) {
@@ -3218,28 +3336,12 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         }
     }
 
-    void ensureSystemTablesMigratedToSystemNamespace(ReadOnlyProps props)
+    void ensureSystemTablesMigratedToSystemNamespace()
             throws SQLException, IOException, IllegalArgumentException, InterruptedException {
-        if (!SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, props)) { return; }
-
-        boolean acquiredMutexLock = false;
-        byte[] mutexRowKey = SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA,
-                PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE);
+        if (!SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, this.getProps())) { return; }
 
         HTableInterface metatable = null;
         try (HBaseAdmin admin = getAdmin()) {
-             // SYSTEM namespace needs to be created via HBase API's because "CREATE SCHEMA" statement tries to write its metadata
-             // in SYSTEM:CATALOG table. Without SYSTEM namespace, SYSTEM:CATALOG table cannot be created.
-            try {
-                ensureNamespaceCreated(QueryConstants.SYSTEM_SCHEMA_NAME);
-            } catch (PhoenixIOException e) {
-                // We could either:
-                // 1) Not access the NS descriptor. The NS may or may not exist at this point.
-                // 2) We could not create the NS
-                // Regardless of the case 1 or 2, if the NS does not exist, we will error expectedly
-                // below. If the NS does exist and is mapped, the below check will exit gracefully.
-            }
-
             List<TableName> tableNames = getSystemTableNamesInDefaultNamespace(admin);
             // No tables exist matching "SYSTEM\..*", they are all already in "SYSTEM:.*"
             if (tableNames.size() == 0) { return; }
@@ -3248,33 +3350,22 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                 logger.warn("Expected 5 system tables but found " + tableNames.size() + ":" + tableNames);
             }
 
-            // Try acquiring a lock in SYSMUTEX table before migrating the tables since it involves disabling the table
-            // If we cannot acquire lock, it means some old client is either migrating SYSCAT or trying to upgrade the
-            // schema of SYSCAT table and hence it should not be interrupted
-            // Create mutex if not already created
-            createSysMutexTableIfNotExists(admin, props);
-            acquiredMutexLock = acquireUpgradeMutex(MetaDataProtocol.MIN_SYSTEM_TABLE_MIGRATION_TIMESTAMP, mutexRowKey);
-            if(acquiredMutexLock) {
-                logger.debug("Acquired lock in SYSMUTEX table for migrating SYSTEM tables to SYSTEM namespace");
-            }
-            // We will not reach here if we fail to acquire the lock, since it throws UpgradeInProgressException
-
             // Handle the upgrade of SYSMUTEX table separately since it doesn't have any entries in SYSCAT
             logger.info("Migrating SYSTEM.MUTEX table to SYSTEM namespace.");
             String sysMutexSrcTableName = PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME;
-            String sysMutexDestTableName = SchemaUtil.getPhysicalName(sysMutexSrcTableName.getBytes(), props).getNameAsString();
+            String sysMutexDestTableName = SchemaUtil.getPhysicalName(sysMutexSrcTableName.getBytes(), this.getProps()).getNameAsString();
             UpgradeUtil.mapTableToNamespace(admin, sysMutexSrcTableName, sysMutexDestTableName, PTableType.SYSTEM);
             tableNames.remove(PhoenixDatabaseMetaData.SYSTEM_MUTEX_HBASE_TABLE_NAME);
 
             byte[] mappedSystemTable = SchemaUtil
-                    .getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES, props).getName();
+                    .getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES, this.getProps()).getName();
             metatable = getTable(mappedSystemTable);
             if (tableNames.contains(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME)) {
                 if (!admin.tableExists(mappedSystemTable)) {
                     logger.info("Migrating SYSTEM.CATALOG table to SYSTEM namespace.");
                     // Actual migration of SYSCAT table
                     UpgradeUtil.mapTableToNamespace(admin, metatable,
-                            PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, props, null, PTableType.SYSTEM,
+                            PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, this.getProps(), null, PTableType.SYSTEM,
                             null);
                     // Invalidate the client-side metadataCache
                     ConnectionQueryServicesImpl.this.removeTable(null,
@@ -3285,7 +3376,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
             }
             for (TableName table : tableNames) {
                 logger.info(String.format("Migrating %s table to SYSTEM namespace.", table.getNameAsString()));
-                UpgradeUtil.mapTableToNamespace(admin, metatable, table.getNameAsString(), props, null, PTableType.SYSTEM,
+                UpgradeUtil.mapTableToNamespace(admin, metatable, table.getNameAsString(), this.getProps(), null, PTableType.SYSTEM,
                         null);
                 ConnectionQueryServicesImpl.this.removeTable(null, table.getNameAsString(), null,
                         MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0);
@@ -3297,9 +3388,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
             if (metatable != null) {
                 metatable.close();
             }
-            if(acquiredMutexLock) {
-                releaseUpgradeMutex(mutexRowKey);
-            }
         }
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 41df7b0..1c56c31 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -242,7 +242,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
     @Override
     public MetaDataMutationResult createTable(List<Mutation> tableMetaData, byte[] physicalName, PTableType tableType,
             Map<String, Object> tableProps, List<Pair<byte[], Map<String, Object>>> families, byte[][] splits,
-            boolean isNamespaceMapped, boolean allocateIndexId) throws SQLException {
+            boolean isNamespaceMapped, boolean allocateIndexId, boolean isDoNotUpgradePropSet) throws SQLException {
         if (tableType == PTableType.INDEX && IndexUtil.isLocalIndexFamily(Bytes.toString(families.iterator().next().getFirst()))) {
             Object dataTableName = tableProps.get(PhoenixDatabaseMetaData.DATA_TABLE_NAME);
             List<HRegionLocation> regionLocations = tableSplits.get(dataTableName);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index e393210..cb7ce58 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -114,9 +114,9 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
     @Override
     public MetaDataMutationResult createTable(List<Mutation> tableMetaData, byte[] physicalName, PTableType tableType,
             Map<String, Object> tableProps, List<Pair<byte[], Map<String, Object>>> families, byte[][] splits,
-            boolean isNamespaceMapped, boolean allocateIndexId) throws SQLException {
+            boolean isNamespaceMapped, boolean allocateIndexId, boolean isDoNotUpgradePropSet) throws SQLException {
         return getDelegate().createTable(tableMetaData, physicalName, tableType, tableProps, families, splits,
-                isNamespaceMapped, allocateIndexId);
+                isNamespaceMapped, allocateIndexId, isDoNotUpgradePropSet);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 1fb668e..b15072a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -2694,7 +2694,8 @@ public class MetaDataClient {
             MetaDataMutationResult result = connection.getQueryServices().createTable(
                     tableMetaData,
                     viewType == ViewType.MAPPED || allocateIndexId ? physicalNames.get(0).getBytes() : null,
-                    tableType, tableProps, familyPropList, splits, isNamespaceMapped, allocateIndexId);
+                    tableType, tableProps, familyPropList, splits, isNamespaceMapped, allocateIndexId,
+                    UpgradeUtil.isNoUpgradeSet(connection.getClientInfo()));
             MutationCode code = result.getMutationCode();
             switch(code) {
             case TABLE_ALREADY_EXISTS:

http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
index b5c3e4a..46907d9 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
@@ -47,9 +47,9 @@ public class ConnectionQueryServicesImplTest {
         ConnectionQueryServicesImpl cqs = mock(ConnectionQueryServicesImpl.class);
         // Invoke the real methods for these two calls
         when(cqs.createSchema(any(List.class), anyString())).thenCallRealMethod();
-        doCallRealMethod().when(cqs).ensureSystemTablesMigratedToSystemNamespace(any(ReadOnlyProps.class));
+        doCallRealMethod().when(cqs).ensureSystemTablesMigratedToSystemNamespace();
         // Do nothing for this method, just check that it was invoked later
-        doNothing().when(cqs).createSysMutexTableIfNotExists(any(HBaseAdmin.class), any(ReadOnlyProps.class));
+        doNothing().when(cqs).createSysMutexTableIfNotExists(any(HBaseAdmin.class));
 
         // Spoof out this call so that ensureSystemTablesUpgrade() will return-fast.
         when(cqs.getSystemTableNamesInDefaultNamespace(any(HBaseAdmin.class))).thenReturn(Collections.<TableName> emptyList());
@@ -60,7 +60,8 @@ public class ConnectionQueryServicesImplTest {
         // Make sure that ensureSystemTablesMigratedToSystemNamespace will try to migrate the system tables.
         Map<String,String> props = new HashMap<>();
         props.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
-        cqs.ensureSystemTablesMigratedToSystemNamespace(new ReadOnlyProps(props));
+        when(cqs.getProps()).thenReturn(new ReadOnlyProps(props));
+        cqs.ensureSystemTablesMigratedToSystemNamespace();
 
         // Should be called after upgradeSystemTables()
         // Proves that execution proceeded

http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-protocol/src/main/MetaDataService.proto
----------------------------------------------------------------------
diff --git a/phoenix-protocol/src/main/MetaDataService.proto b/phoenix-protocol/src/main/MetaDataService.proto
index 2ba2b4c..369522c 100644
--- a/phoenix-protocol/src/main/MetaDataService.proto
+++ b/phoenix-protocol/src/main/MetaDataService.proto
@@ -168,6 +168,7 @@ message GetVersionRequest {
 
 message GetVersionResponse {
   required int64 version = 1;
+  optional int64 systemCatalogTimestamp = 2;
 }
 
 message ClearTableFromCacheRequest {


[42/50] [abbrv] phoenix git commit: PHOENIX-4707 Include python driver in assembly

Posted by td...@apache.org.
PHOENIX-4707 Include python driver in assembly


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/4b841996
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/4b841996
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/4b841996

Branch: refs/heads/system-catalog
Commit: 4b84199681f241843855ce652e582e06bb99c1cf
Parents: ff32734
Author: Josh Elser <el...@apache.org>
Authored: Thu Apr 26 10:00:16 2018 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Apr 26 15:25:47 2018 -0400

----------------------------------------------------------------------
 phoenix-assembly/src/build/components/all-common-files.xml | 5 +++++
 phoenix-assembly/src/build/src.xml                         | 7 ++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/4b841996/phoenix-assembly/src/build/components/all-common-files.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/components/all-common-files.xml b/phoenix-assembly/src/build/components/all-common-files.xml
index af7888d..a17a138 100644
--- a/phoenix-assembly/src/build/components/all-common-files.xml
+++ b/phoenix-assembly/src/build/components/all-common-files.xml
@@ -86,5 +86,10 @@
           <fileMode>0644</fileMode>
           <directoryMode>0755</directoryMode>
       </fileSet>
+    <fileSet>
+      <directory>${project.basedir}/../python</directory>
+      <fileMode>0644</fileMode>
+      <directoryMode>0755</directoryMode>
+    </fileSet>
   </fileSets>
 </component>

http://git-wip-us.apache.org/repos/asf/phoenix/blob/4b841996/phoenix-assembly/src/build/src.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/src.xml b/phoenix-assembly/src/build/src.xml
index 5dc92a8..cfa73d6 100644
--- a/phoenix-assembly/src/build/src.xml
+++ b/phoenix-assembly/src/build/src.xml
@@ -112,11 +112,16 @@
       <fileMode>0644</fileMode>
       <directoryMode>0755</directoryMode>
     </fileSet>
-      <fileSet>
+    <fileSet>
       <directory>${project.basedir}/../examples</directory>
       <fileMode>0644</fileMode>
       <directoryMode>0755</directoryMode>
     </fileSet>
+    <fileSet>
+      <directory>${project.basedir}/../python</directory>
+      <fileMode>0644</fileMode>
+      <directoryMode>0755</directoryMode>
+    </fileSet>
   </fileSets>
 
 </assembly>


[43/50] [abbrv] phoenix git commit: fix test failures

Posted by td...@apache.org.
fix test failures


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/0f58bc92
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/0f58bc92
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/0f58bc92

Branch: refs/heads/system-catalog
Commit: 0f58bc92db3c6101361b6a9b6cffaf340865a856
Parents: f8ce60c
Author: Thomas D'Silva <td...@apache.org>
Authored: Fri Apr 27 10:16:28 2018 -0700
Committer: Thomas D'Silva <td...@apache.org>
Committed: Fri Apr 27 10:16:28 2018 -0700

----------------------------------------------------------------------
 .../MigrateSystemTablesToSystemNamespaceIT.java |  2 +-
 .../SystemCatalogCreationOnConnectionIT.java    | 34 +++++++++++++-------
 .../org/apache/phoenix/schema/PTableImpl.java   |  2 +-
 .../org/apache/phoenix/util/UpgradeUtil.java    |  3 ++
 4 files changed, 28 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/0f58bc92/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
index 7960543..a4350ef 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
@@ -63,7 +63,7 @@ public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest {
 
     private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
             "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
-            "SYSTEM.MUTEX","SYSTEM.LOG"));
+            "SYSTEM.MUTEX","SYSTEM.LOG", "SYSTEM.CHILD_LINK"));
     private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
             Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
                     "SYSTEM:MUTEX","SYSTEM:LOG", "SYSTEM:CHILD_LINK"));

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0f58bc92/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
index 689eb20..aa2d971 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
@@ -17,6 +17,22 @@
  */
 package org.apache.phoenix.end2end;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeoutException;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
@@ -28,7 +44,11 @@ import org.apache.phoenix.exception.UpgradeRequiredException;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver;
 import org.apache.phoenix.jdbc.PhoenixTestDriver;
-import org.apache.phoenix.query.*;
+import org.apache.phoenix.query.ConnectionQueryServices;
+import org.apache.phoenix.query.ConnectionQueryServicesImpl;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesTestImpl;
 import org.apache.phoenix.util.ReadOnlyProps;
 import org.apache.phoenix.util.UpgradeUtil;
 import org.junit.After;
@@ -36,14 +56,6 @@ import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.util.*;
-import java.util.concurrent.TimeoutException;
-
-import static org.junit.Assert.*;
-
 @Category(NeedsOwnMiniClusterTest.class)
 public class SystemCatalogCreationOnConnectionIT {
     private HBaseTestingUtility testUtil = null;
@@ -60,11 +72,11 @@ public class SystemCatalogCreationOnConnectionIT {
 
     private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
       "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
-      "SYSTEM.MUTEX", "SYSTEM.LOG"));
+      "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK"));
 
     private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
       Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
-        "SYSTEM:MUTEX", "SYSTEM:LOG"));
+        "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK"));
 
     private static class PhoenixSysCatCreationServices extends ConnectionQueryServicesImpl {
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0f58bc92/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index 76d6267..b5304dd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -308,7 +308,7 @@ public class PTableImpl implements PTable {
             table.getIndexes(), basePTable.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
             table.isWALDisabled(), basePTable.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
             baseTableColumnCount, table.rowKeyOrderOptimizable(), basePTable.getTransactionProvider(), table.getUpdateCacheFrequency(),
-            table.getIndexDisableTimestamp(), basePTable.isNamespaceMapped(), basePTable.getAutoPartitionSeqName(), basePTable.isAppendOnlySchema(),
+            table.getIndexDisableTimestamp(), table.isNamespaceMapped(), basePTable.getAutoPartitionSeqName(), basePTable.isAppendOnlySchema(),
             basePTable.getImmutableStorageScheme(), basePTable.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
     }
     

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0f58bc92/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
index d41b0cb..db5d3c5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
@@ -1908,6 +1908,9 @@ public class UpgradeUtil {
             mapTableToNamespace(admin, metatable, tableName, newPhysicalTablename, readOnlyProps,
                     PhoenixRuntime.getCurrentScn(readOnlyProps), tableName, table.getType(),conn.getTenantId());
             // clear the cache and get new table
+            conn.removeTable(conn.getTenantId(), tableName,
+                table.getParentName() != null ? table.getParentName().getString() : null,
+                table.getTimeStamp());
             conn.getQueryServices().clearTableFromCache(
                     conn.getTenantId() == null ? ByteUtil.EMPTY_BYTE_ARRAY : conn.getTenantId().getBytes(),
                     table.getSchemaName().getBytes(), table.getTableName().getBytes(),


[35/50] [abbrv] phoenix git commit: fix test failures

Posted by td...@apache.org.
fix test failures


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/f8ce60c0
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/f8ce60c0
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/f8ce60c0

Branch: refs/heads/system-catalog
Commit: f8ce60c0655f012d01db4a863c0734475b2b2973
Parents: d615dc0
Author: Thomas D'Silva <td...@apache.org>
Authored: Wed Apr 25 14:45:33 2018 -0700
Committer: Thomas D'Silva <td...@apache.org>
Committed: Wed Apr 25 14:45:33 2018 -0700

----------------------------------------------------------------------
 .../query/ConnectionQueryServicesImpl.java      |   9 +-
 .../query/ConnectionlessQueryServicesImpl.java  |   6 +-
 .../apache/phoenix/query/QueryConstants.java    | 406 ++++++++-----------
 .../org/apache/phoenix/schema/PTableImpl.java   |   2 +-
 4 files changed, 191 insertions(+), 232 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/f8ce60c0/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index c453e13..45016c6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -2473,6 +2473,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     protected String getLogTableDDL() {
         return setSystemLogDDLProperties(QueryConstants.CREATE_LOG_METADATA);
     }
+    
+    // Available for testing
+    protected String getChildLinkDDL() {
+        return setSystemDDLProperties(QueryConstants.CREATE_CHILD_LINK_METADATA);
+    }
 
     private String setSystemDDLProperties(String ddl) {
         return String.format(ddl,
@@ -2693,7 +2698,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
             metaConnection.createStatement().execute(getLogTableDDL());
         } catch (TableAlreadyExistsException ignore) {}
         try {
-            metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_CHILD_LINK_METADATA);
+            metaConnection.createStatement().executeUpdate(getChildLinkDDL());
         } catch (TableAlreadyExistsException e) {}
         // Catch the IOException to log the error message and then bubble it up for the client to retry.
         try {
@@ -3129,7 +3134,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                 metaConnection.createStatement().executeUpdate(getLogTableDDL());
             } catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {}
             try {
-                metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_CHILD_LINK_METADATA);
+                metaConnection.createStatement().executeUpdate(getChildLinkDDL());
             } catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {}
 
             // In case namespace mapping is enabled and system table to system namespace mapping is also enabled,

http://git-wip-us.apache.org/repos/asf/phoenix/blob/f8ce60c0/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 21f4f74..0fdf5e6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -166,6 +166,10 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
     protected String getLogTableDDL() {
         return setSystemLogDDLProperties(QueryConstants.CREATE_LOG_METADATA);
     }
+    
+    protected String getChildLinkDDL() {
+        return setSystemDDLProperties(QueryConstants.CREATE_CHILD_LINK_METADATA);
+    }
 
     private String setSystemDDLProperties(String ddl) {
         return String.format(ddl,
@@ -367,7 +371,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
                 } catch (NewerTableAlreadyExistsException ignore) {}
                 try {
                     metaConnection.createStatement()
-                            .executeUpdate(QueryConstants.CREATE_CHILD_LINK_METADATA);
+                            .executeUpdate(getChildLinkDDL());
                 } catch (NewerTableAlreadyExistsException ignore) {
                 }
             } catch (SQLException e) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/f8ce60c0/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index c7fa438..aa37517 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -1,23 +1,15 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
+ * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under the License.
  */
 package org.apache.phoenix.query;
 
-
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.*;
 
 import java.math.BigDecimal;
@@ -36,12 +28,8 @@ import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.SortOrder;
 import org.apache.phoenix.schema.TableProperty;
 
-
 /**
- *
  * Constants used during querying
- *
- *
  * @since 0.1
  */
 public interface QueryConstants {
@@ -57,110 +45,125 @@ public interface QueryConstants {
     public static final String NULL_DISPLAY_TEXT = "<null>";
     public static final long UNSET_TIMESTAMP = -1;
 
-    public enum JoinType {INNER, LEFT_OUTER}
+    public enum JoinType {
+        INNER, LEFT_OUTER
+    }
+
     public final static String SYSTEM_SCHEMA_NAME = "SYSTEM";
     public final static byte[] SYSTEM_SCHEMA_NAME_BYTES = Bytes.toBytes(SYSTEM_SCHEMA_NAME);
     public final static String PHOENIX_METADATA = "table";
     public final static String OFFSET_ROW_KEY = "_OFFSET_";
     public final static byte[] OFFSET_ROW_KEY_BYTES = Bytes.toBytes(OFFSET_ROW_KEY);
-    public final static ImmutableBytesPtr OFFSET_ROW_KEY_PTR = new ImmutableBytesPtr(OFFSET_ROW_KEY_BYTES);
+    public final static ImmutableBytesPtr OFFSET_ROW_KEY_PTR =
+            new ImmutableBytesPtr(OFFSET_ROW_KEY_BYTES);
 
     public static final long AGG_TIMESTAMP = HConstants.LATEST_TIMESTAMP;
     /**
      * Key used for a single row aggregation where there is no group by
      */
     public final static byte[] UNGROUPED_AGG_ROW_KEY = Bytes.toBytes("a");
-    
+
     /** BEGIN Set of reserved column qualifiers **/
-    
+
     public static final String RESERVED_COLUMN_FAMILY = "_v";
     public static final byte[] RESERVED_COLUMN_FAMILY_BYTES = Bytes.toBytes(RESERVED_COLUMN_FAMILY);
-    
+
     public static final byte[] VALUE_COLUMN_FAMILY = RESERVED_COLUMN_FAMILY_BYTES;
-    public static final byte[] VALUE_COLUMN_QUALIFIER = QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS.encode(1);
-    
+    public static final byte[] VALUE_COLUMN_QUALIFIER =
+            QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS.encode(1);
+
     public static final byte[] ARRAY_VALUE_COLUMN_FAMILY = RESERVED_COLUMN_FAMILY_BYTES;
-    public static final byte[] ARRAY_VALUE_COLUMN_QUALIFIER = QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS.encode(2);
-    
+    public static final byte[] ARRAY_VALUE_COLUMN_QUALIFIER =
+            QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS.encode(2);
+
     public final static PName SINGLE_COLUMN_NAME = PNameFactory.newNormalizedName("s");
     public final static PName SINGLE_COLUMN_FAMILY_NAME = PNameFactory.newNormalizedName("s");
     public final static byte[] SINGLE_COLUMN = SINGLE_COLUMN_NAME.getBytes();
     public final static byte[] SINGLE_COLUMN_FAMILY = SINGLE_COLUMN_FAMILY_NAME.getBytes();
 
     /** END Set of reserved column qualifiers **/
-    
-    public static final byte[] TRUE = new byte[] {1};
-    
+
+    public static final byte[] TRUE = new byte[] { 1 };
+
     /**
-     * The priority property for an hbase table. This is already in HTD, but older versions of
-     * HBase do not have this, so we re-defined it here. Once Phoenix is HBase-1.3+, we can remote.
+     * The priority property for an hbase table. This is already in HTD, but older versions of HBase
+     * do not have this, so we re-defined it here. Once Phoenix is HBase-1.3+, we can remote.
      */
     public static final String PRIORITY = "PRIORITY";
 
     /**
-     * Separator used between variable length keys for a composite key.
-     * Variable length data types may not use this byte value.
+     * Separator used between variable length keys for a composite key. Variable length data types
+     * may not use this byte value.
      */
     public static final byte SEPARATOR_BYTE = (byte) 0;
-    public static final byte[] SEPARATOR_BYTE_ARRAY = new byte[] {SEPARATOR_BYTE};
+    public static final byte[] SEPARATOR_BYTE_ARRAY = new byte[] { SEPARATOR_BYTE };
     public static final byte DESC_SEPARATOR_BYTE = SortOrder.invert(SEPARATOR_BYTE);
-    public static final byte[] DESC_SEPARATOR_BYTE_ARRAY = new byte[] {DESC_SEPARATOR_BYTE};
+    public static final byte[] DESC_SEPARATOR_BYTE_ARRAY = new byte[] { DESC_SEPARATOR_BYTE };
 
     public static final String DEFAULT_COPROCESS_PATH = "phoenix.jar";
     public static final String DEFAULT_COPROCESS_JAR_NAME = "phoenix-[version]-server.jar";
-    
+
     public final static int MILLIS_IN_DAY = 1000 * 60 * 60 * 24;
 
     public static final String EMPTY_COLUMN_NAME = "_0";
     // For transactional tables, the value of our empty key value can no longer be empty
     // since empty values are treated as column delete markers.
     public static final byte[] EMPTY_COLUMN_BYTES = Bytes.toBytes(EMPTY_COLUMN_NAME);
-    public static final ImmutableBytesPtr EMPTY_COLUMN_BYTES_PTR = new ImmutableBytesPtr(
-            EMPTY_COLUMN_BYTES);
+    public static final ImmutableBytesPtr EMPTY_COLUMN_BYTES_PTR =
+            new ImmutableBytesPtr(EMPTY_COLUMN_BYTES);
     public static final Integer ENCODED_EMPTY_COLUMN_NAME = 0;
-    public static final byte[] ENCODED_EMPTY_COLUMN_BYTES = QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS.encode(ENCODED_EMPTY_COLUMN_NAME);
+    public static final byte[] ENCODED_EMPTY_COLUMN_BYTES =
+            QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS.encode(ENCODED_EMPTY_COLUMN_NAME);
     public final static String EMPTY_COLUMN_VALUE = "x";
     public final static byte[] EMPTY_COLUMN_VALUE_BYTES = Bytes.toBytes(EMPTY_COLUMN_VALUE);
-    public static final ImmutableBytesPtr EMPTY_COLUMN_VALUE_BYTES_PTR = new ImmutableBytesPtr(
-            EMPTY_COLUMN_VALUE_BYTES);
+    public static final ImmutableBytesPtr EMPTY_COLUMN_VALUE_BYTES_PTR =
+            new ImmutableBytesPtr(EMPTY_COLUMN_VALUE_BYTES);
     public static final String ENCODED_EMPTY_COLUMN_VALUE = EMPTY_COLUMN_VALUE;
     public final static byte[] ENCODED_EMPTY_COLUMN_VALUE_BYTES = Bytes.toBytes(EMPTY_COLUMN_VALUE);
-    public static final ImmutableBytesPtr ENCODED_EMPTY_COLUMN_VALUE_BYTES_PTR = new ImmutableBytesPtr(
-            ENCODED_EMPTY_COLUMN_VALUE_BYTES);
+    public static final ImmutableBytesPtr ENCODED_EMPTY_COLUMN_VALUE_BYTES_PTR =
+            new ImmutableBytesPtr(ENCODED_EMPTY_COLUMN_VALUE_BYTES);
     public static final String DEFAULT_COLUMN_FAMILY = "0";
     public static final byte[] DEFAULT_COLUMN_FAMILY_BYTES = Bytes.toBytes(DEFAULT_COLUMN_FAMILY);
-    public static final ImmutableBytesPtr DEFAULT_COLUMN_FAMILY_BYTES_PTR = new ImmutableBytesPtr(
-            DEFAULT_COLUMN_FAMILY_BYTES);
-    // column qualifier of the single key value used to store all columns for the COLUMNS_STORED_IN_SINGLE_CELL storage scheme
+    public static final ImmutableBytesPtr DEFAULT_COLUMN_FAMILY_BYTES_PTR =
+            new ImmutableBytesPtr(DEFAULT_COLUMN_FAMILY_BYTES);
+    // column qualifier of the single key value used to store all columns for the
+    // COLUMNS_STORED_IN_SINGLE_CELL storage scheme
     public static final String SINGLE_KEYVALUE_COLUMN_QUALIFIER = "1";
-    public final static byte[] SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES = Bytes.toBytes(SINGLE_KEYVALUE_COLUMN_QUALIFIER);
-    public static final ImmutableBytesPtr SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES_PTR = new ImmutableBytesPtr(
-            SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES);
+    public final static byte[] SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES =
+            Bytes.toBytes(SINGLE_KEYVALUE_COLUMN_QUALIFIER);
+    public static final ImmutableBytesPtr SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES_PTR =
+            new ImmutableBytesPtr(SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES);
 
     public static final String LOCAL_INDEX_COLUMN_FAMILY_PREFIX = "L#";
-    public static final byte[] LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES = Bytes.toBytes(LOCAL_INDEX_COLUMN_FAMILY_PREFIX);
-    public static final ImmutableBytesPtr LOCAL_INDEX_COLUMN_FAMILY_PREFIX_PTR = new ImmutableBytesPtr(
-        LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES);
-    
-    public static final String DEFAULT_LOCAL_INDEX_COLUMN_FAMILY = LOCAL_INDEX_COLUMN_FAMILY_PREFIX + DEFAULT_COLUMN_FAMILY;
-    public static final byte[] DEFAULT_LOCAL_INDEX_COLUMN_FAMILY_BYTES = Bytes.toBytes(DEFAULT_LOCAL_INDEX_COLUMN_FAMILY);
-    public static final ImmutableBytesPtr DEFAULT_LOCAL_INDEX_COLUMN_FAMILY_BYTES_PTR = new ImmutableBytesPtr(
-               DEFAULT_LOCAL_INDEX_COLUMN_FAMILY_BYTES);
+    public static final byte[] LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES =
+            Bytes.toBytes(LOCAL_INDEX_COLUMN_FAMILY_PREFIX);
+    public static final ImmutableBytesPtr LOCAL_INDEX_COLUMN_FAMILY_PREFIX_PTR =
+            new ImmutableBytesPtr(LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES);
+
+    public static final String DEFAULT_LOCAL_INDEX_COLUMN_FAMILY =
+            LOCAL_INDEX_COLUMN_FAMILY_PREFIX + DEFAULT_COLUMN_FAMILY;
+    public static final byte[] DEFAULT_LOCAL_INDEX_COLUMN_FAMILY_BYTES =
+            Bytes.toBytes(DEFAULT_LOCAL_INDEX_COLUMN_FAMILY);
+    public static final ImmutableBytesPtr DEFAULT_LOCAL_INDEX_COLUMN_FAMILY_BYTES_PTR =
+            new ImmutableBytesPtr(DEFAULT_LOCAL_INDEX_COLUMN_FAMILY_BYTES);
 
     public static final String ALL_FAMILY_PROPERTIES_KEY = "";
     public static final String SYSTEM_TABLE_PK_NAME = "pk";
 
     public static final double MILLIS_TO_NANOS_CONVERTOR = Math.pow(10, 6);
-    public static final BigDecimal BD_MILLIS_NANOS_CONVERSION = BigDecimal.valueOf(MILLIS_TO_NANOS_CONVERTOR);
-    public static final BigDecimal BD_MILLIS_IN_DAY = BigDecimal.valueOf(QueryConstants.MILLIS_IN_DAY);
+    public static final BigDecimal BD_MILLIS_NANOS_CONVERSION =
+            BigDecimal.valueOf(MILLIS_TO_NANOS_CONVERTOR);
+    public static final BigDecimal BD_MILLIS_IN_DAY =
+            BigDecimal.valueOf(QueryConstants.MILLIS_IN_DAY);
     public static final int MAX_ALLOWED_NANOS = 999999999;
     public static final int NANOS_IN_SECOND = BigDecimal.valueOf(Math.pow(10, 9)).intValue();
     public static final int DIVERGED_VIEW_BASE_COLUMN_COUNT = -100;
     public static final int BASE_TABLE_BASE_COLUMN_COUNT = -1;
-    
+
     /**
-     * We mark counter values 0 to 10 as reserved. Value 0 is used by {@link #ENCODED_EMPTY_COLUMN_NAME}. Values 1-10
-     * are reserved for special column qualifiers returned by Phoenix co-processors.
+     * We mark counter values 0 to 10 as reserved. Value 0 is used by
+     * {@link #ENCODED_EMPTY_COLUMN_NAME}. Values 1-10 are reserved for special column qualifiers
+     * returned by Phoenix co-processors.
      */
     public static final int ENCODED_CQ_COUNTER_INITIAL_VALUE = 11;
     public static final String CREATE_TABLE_METADATA =
@@ -168,190 +171,137 @@ public interface QueryConstants {
             // exception and add columns to the SYSTEM.TABLE dynamically.
             "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"(\n" +
             // PK columns
-            TENANT_ID + " VARCHAR NULL," +
-            TABLE_SCHEM + " VARCHAR NULL," +
-            TABLE_NAME + " VARCHAR NOT NULL," +
-            COLUMN_NAME + " VARCHAR NULL," + // null for table row
-            COLUMN_FAMILY + " VARCHAR NULL," + // using for CF to uniqueness for columns
-            // Table metadata (will be null for column rows)
-            TABLE_SEQ_NUM + " BIGINT," +
-            TABLE_TYPE + " CHAR(1)," +
-            PK_NAME + " VARCHAR," +
-            COLUMN_COUNT + " INTEGER," +
-            SALT_BUCKETS + " INTEGER," +
-            DATA_TABLE_NAME + " VARCHAR," +
-            INDEX_STATE + " CHAR(1),\n" +
-            IMMUTABLE_ROWS + " BOOLEAN,\n" +
-            VIEW_STATEMENT + " VARCHAR,\n" +
-            DEFAULT_COLUMN_FAMILY_NAME + " VARCHAR,\n" +
-            DISABLE_WAL + " BOOLEAN,\n" +
-            MULTI_TENANT + " BOOLEAN,\n" +
-            VIEW_TYPE + " UNSIGNED_TINYINT,\n" +
-            VIEW_INDEX_ID + " SMALLINT,\n" +
-            // Column metadata (will be null for table row)
-            DATA_TYPE + " INTEGER," +
-            COLUMN_SIZE + " INTEGER," +
-            DECIMAL_DIGITS + " INTEGER," +
-            NULLABLE + " INTEGER," +
-            ORDINAL_POSITION + " INTEGER," +
-            SORT_ORDER + " INTEGER," +
-            ARRAY_SIZE + " INTEGER,\n" +
-            VIEW_CONSTANT + " VARBINARY,\n" +
-            IS_VIEW_REFERENCED + " BOOLEAN,\n" +
-            KEY_SEQ + " SMALLINT,\n" +
-            // Link metadata (only set on rows linking table to index or view)
-            LINK_TYPE + " UNSIGNED_TINYINT,\n" +
-            // Unused
-            TYPE_NAME + " VARCHAR," +
-            REMARKS + " VARCHAR," +
-            SELF_REFERENCING_COL_NAME + " VARCHAR," +
-            REF_GENERATION + " VARCHAR," +
-            BUFFER_LENGTH + " INTEGER," +
-            NUM_PREC_RADIX + " INTEGER," +
-            COLUMN_DEF + " VARCHAR," +
-            SQL_DATA_TYPE + " INTEGER," +
-            SQL_DATETIME_SUB + " INTEGER," +
-            CHAR_OCTET_LENGTH + " INTEGER," +
-            IS_NULLABLE + " VARCHAR," +
-            SCOPE_CATALOG + " VARCHAR," +
-            SCOPE_SCHEMA + " VARCHAR," +
-            SCOPE_TABLE + " VARCHAR," +
-            SOURCE_DATA_TYPE + " SMALLINT," +
-            IS_AUTOINCREMENT + " VARCHAR," +
-            INDEX_TYPE + " UNSIGNED_TINYINT," +
-            INDEX_DISABLE_TIMESTAMP + " BIGINT," +
-            STORE_NULLS + " BOOLEAN," +
-            BASE_COLUMN_COUNT + " INTEGER," +
-            // Column metadata (will be null for table row)
-            IS_ROW_TIMESTAMP + " BOOLEAN, " +
-            TRANSACTIONAL + " BOOLEAN," +
-            UPDATE_CACHE_FREQUENCY + " BIGINT," +
-            IS_NAMESPACE_MAPPED + " BOOLEAN," +
-            AUTO_PARTITION_SEQ + " VARCHAR," +
-            APPEND_ONLY_SCHEMA + " BOOLEAN," +
-            GUIDE_POSTS_WIDTH + " BIGINT," +
-            COLUMN_QUALIFIER + " VARBINARY," +
-            IMMUTABLE_STORAGE_SCHEME + " TINYINT, " +
-            ENCODING_SCHEME + " TINYINT, " +
-            COLUMN_QUALIFIER_COUNTER + " INTEGER, " +
-            USE_STATS_FOR_PARALLELIZATION + " BOOLEAN, " +
-            TRANSACTION_PROVIDER + " TINYINT, " +
-            "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ","
-            + TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" +
-            HConstants.VERSIONS + "=%s,\n" +
-            HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n" +
-            // Install split policy to prevent a tenant's metadata from being split across regions.
-            HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + 
-            PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
+                    TENANT_ID + " VARCHAR NULL," + TABLE_SCHEM + " VARCHAR NULL," + TABLE_NAME
+                    + " VARCHAR NOT NULL," + COLUMN_NAME + " VARCHAR NULL," + // null for table row
+                    COLUMN_FAMILY + " VARCHAR NULL," + // using for CF to uniqueness for columns
+                    // Table metadata (will be null for column rows)
+                    TABLE_SEQ_NUM + " BIGINT," + TABLE_TYPE + " CHAR(1)," + PK_NAME + " VARCHAR,"
+                    + COLUMN_COUNT + " INTEGER," + SALT_BUCKETS + " INTEGER," + DATA_TABLE_NAME
+                    + " VARCHAR," + INDEX_STATE + " CHAR(1),\n" + IMMUTABLE_ROWS + " BOOLEAN,\n"
+                    + VIEW_STATEMENT + " VARCHAR,\n" + DEFAULT_COLUMN_FAMILY_NAME + " VARCHAR,\n"
+                    + DISABLE_WAL + " BOOLEAN,\n" + MULTI_TENANT + " BOOLEAN,\n" + VIEW_TYPE
+                    + " UNSIGNED_TINYINT,\n" + VIEW_INDEX_ID + " SMALLINT,\n" +
+                    // Column metadata (will be null for table row)
+                    DATA_TYPE + " INTEGER," + COLUMN_SIZE + " INTEGER," + DECIMAL_DIGITS
+                    + " INTEGER," + NULLABLE + " INTEGER," + ORDINAL_POSITION + " INTEGER,"
+                    + SORT_ORDER + " INTEGER," + ARRAY_SIZE + " INTEGER,\n" + VIEW_CONSTANT
+                    + " VARBINARY,\n" + IS_VIEW_REFERENCED + " BOOLEAN,\n" + KEY_SEQ
+                    + " SMALLINT,\n" +
+                    // Link metadata (only set on rows linking table to index or view)
+                    LINK_TYPE + " UNSIGNED_TINYINT,\n" +
+                    // Unused
+                    TYPE_NAME + " VARCHAR," + REMARKS + " VARCHAR," + SELF_REFERENCING_COL_NAME
+                    + " VARCHAR," + REF_GENERATION + " VARCHAR," + BUFFER_LENGTH + " INTEGER,"
+                    + NUM_PREC_RADIX + " INTEGER," + COLUMN_DEF + " VARCHAR," + SQL_DATA_TYPE
+                    + " INTEGER," + SQL_DATETIME_SUB + " INTEGER," + CHAR_OCTET_LENGTH + " INTEGER,"
+                    + IS_NULLABLE + " VARCHAR," + SCOPE_CATALOG + " VARCHAR," + SCOPE_SCHEMA
+                    + " VARCHAR," + SCOPE_TABLE + " VARCHAR," + SOURCE_DATA_TYPE + " SMALLINT,"
+                    + IS_AUTOINCREMENT + " VARCHAR," + INDEX_TYPE + " UNSIGNED_TINYINT,"
+                    + INDEX_DISABLE_TIMESTAMP + " BIGINT," + STORE_NULLS + " BOOLEAN,"
+                    + BASE_COLUMN_COUNT + " INTEGER," +
+                    // Column metadata (will be null for table row)
+                    IS_ROW_TIMESTAMP + " BOOLEAN, " + TRANSACTIONAL + " BOOLEAN,"
+                    + UPDATE_CACHE_FREQUENCY + " BIGINT," + IS_NAMESPACE_MAPPED + " BOOLEAN,"
+                    + AUTO_PARTITION_SEQ + " VARCHAR," + APPEND_ONLY_SCHEMA + " BOOLEAN,"
+                    + GUIDE_POSTS_WIDTH + " BIGINT," + COLUMN_QUALIFIER + " VARBINARY,"
+                    + IMMUTABLE_STORAGE_SCHEME + " TINYINT, " + ENCODING_SCHEME + " TINYINT, "
+                    + COLUMN_QUALIFIER_COUNTER + " INTEGER, " + USE_STATS_FOR_PARALLELIZATION
+                    + " BOOLEAN, " + TRANSACTION_PROVIDER + " TINYINT, " + "CONSTRAINT "
+                    + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + TABLE_SCHEM + ","
+                    + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n"
+                    + HConstants.VERSIONS + "=%s,\n" + HColumnDescriptor.KEEP_DELETED_CELLS
+                    + "=%s,\n" +
+                    // Install split policy to prevent a tenant's metadata from being split across
+                    // regions.
+                    HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName()
+                    + "',\n" + PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
 
     public static final String CREATE_STATS_TABLE_METADATA =
             "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_STATS_TABLE + "\"(\n" +
             // PK columns
-            PHYSICAL_NAME  + " VARCHAR NOT NULL," +
-            COLUMN_FAMILY + " VARCHAR," +
-            GUIDE_POST_KEY  + " VARBINARY," +
-            GUIDE_POSTS_WIDTH + " BIGINT," +
-            LAST_STATS_UPDATE_TIME+ " DATE, "+
-            GUIDE_POSTS_ROW_COUNT+ " BIGINT, "+
-            "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY ("
-            + PHYSICAL_NAME + ","
-            + COLUMN_FAMILY + ","+ GUIDE_POST_KEY+"))\n" +
-            // Install split policy to prevent a physical table's stats from being split across regions.
-            HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + 
-            PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
+                    PHYSICAL_NAME + " VARCHAR NOT NULL," + COLUMN_FAMILY + " VARCHAR,"
+                    + GUIDE_POST_KEY + " VARBINARY," + GUIDE_POSTS_WIDTH + " BIGINT,"
+                    + LAST_STATS_UPDATE_TIME + " DATE, " + GUIDE_POSTS_ROW_COUNT + " BIGINT, "
+                    + "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + PHYSICAL_NAME + ","
+                    + COLUMN_FAMILY + "," + GUIDE_POST_KEY + "))\n" +
+                    // Install split policy to prevent a physical table's stats from being split
+                    // across regions.
+                    HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName()
+                    + "',\n" + PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
 
     public static final String CREATE_SEQUENCE_METADATA =
-            "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + TYPE_SEQUENCE + "\"(\n" +
-            TENANT_ID + " VARCHAR NULL," +
-            SEQUENCE_SCHEMA + " VARCHAR NULL, \n" +
-            SEQUENCE_NAME +  " VARCHAR NOT NULL, \n" +
-            START_WITH + " BIGINT, \n" +
-            CURRENT_VALUE + " BIGINT, \n" +
-            INCREMENT_BY  + " BIGINT, \n" +
-            CACHE_SIZE  + " BIGINT, \n" +
-            //  the following three columns were added in 3.1/4.1
-            MIN_VALUE + " BIGINT, \n" +
-            MAX_VALUE + " BIGINT, \n" +
-            CYCLE_FLAG + " BOOLEAN, \n" +
-            LIMIT_REACHED_FLAG + " BOOLEAN \n" +
-            " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + SEQUENCE_SCHEMA + "," + SEQUENCE_NAME + "))\n" +
-            PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
+            "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + TYPE_SEQUENCE + "\"(\n" + TENANT_ID
+                    + " VARCHAR NULL," + SEQUENCE_SCHEMA + " VARCHAR NULL, \n" + SEQUENCE_NAME
+                    + " VARCHAR NOT NULL, \n" + START_WITH + " BIGINT, \n" + CURRENT_VALUE
+                    + " BIGINT, \n" + INCREMENT_BY + " BIGINT, \n" + CACHE_SIZE + " BIGINT, \n" +
+                    // the following three columns were added in 3.1/4.1
+                    MIN_VALUE + " BIGINT, \n" + MAX_VALUE + " BIGINT, \n" + CYCLE_FLAG
+                    + " BOOLEAN, \n" + LIMIT_REACHED_FLAG + " BOOLEAN \n" + " CONSTRAINT "
+                    + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + SEQUENCE_SCHEMA
+                    + "," + SEQUENCE_NAME + "))\n" + PhoenixDatabaseMetaData.TRANSACTIONAL + "="
+                    + Boolean.FALSE;
     public static final String CREATE_SYSTEM_SCHEMA = "CREATE SCHEMA " + SYSTEM_CATALOG_SCHEMA;
     public static final String UPGRADE_TABLE_SNAPSHOT_PREFIX = "_UPGRADING_TABLE_";
 
     public static final String CREATE_FUNCTION_METADATA =
             "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_FUNCTION_TABLE + "\"(\n" +
-             // Pk columns
-            TENANT_ID + " VARCHAR NULL," +
-            FUNCTION_NAME + " VARCHAR NOT NULL, \n" +
-            NUM_ARGS + " INTEGER, \n" +
-            // Function metadata (will be null for argument row)
-            CLASS_NAME +  " VARCHAR, \n" +
-            JAR_PATH + "  VARCHAR, \n" +
-            RETURN_TYPE + " VARCHAR, \n" +
-            // Argument metadata (will be null for function row)
-            TYPE + " VARCHAR, \n" +
-            ARG_POSITION + " VARBINARY, \n" +
-            IS_ARRAY + " BOOLEAN, \n" +
-            IS_CONSTANT + " BOOLEAN, \n" +
-            DEFAULT_VALUE + " VARCHAR, \n" +
-            MIN_VALUE + " VARCHAR, \n" +
-            MAX_VALUE + " VARCHAR, \n" +
-            " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ", " + FUNCTION_NAME + ", " + TYPE + ", " + ARG_POSITION + "))\n" +
-            HConstants.VERSIONS + "=%s,\n" +
-            HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n"+
-            // Install split policy to prevent a tenant's metadata from being split across regions.
-            HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + 
-            PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
-    
+            // Pk columns
+                    TENANT_ID + " VARCHAR NULL," + FUNCTION_NAME + " VARCHAR NOT NULL, \n"
+                    + NUM_ARGS + " INTEGER, \n" +
+                    // Function metadata (will be null for argument row)
+                    CLASS_NAME + " VARCHAR, \n" + JAR_PATH + "  VARCHAR, \n" + RETURN_TYPE
+                    + " VARCHAR, \n" +
+                    // Argument metadata (will be null for function row)
+                    TYPE + " VARCHAR, \n" + ARG_POSITION + " VARBINARY, \n" + IS_ARRAY
+                    + " BOOLEAN, \n" + IS_CONSTANT + " BOOLEAN, \n" + DEFAULT_VALUE + " VARCHAR, \n"
+                    + MIN_VALUE + " VARCHAR, \n" + MAX_VALUE + " VARCHAR, \n" + " CONSTRAINT "
+                    + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ", " + FUNCTION_NAME
+                    + ", " + TYPE + ", " + ARG_POSITION + "))\n" + HConstants.VERSIONS + "=%s,\n"
+                    + HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n" +
+                    // Install split policy to prevent a tenant's metadata from being split across
+                    // regions.
+                    HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName()
+                    + "',\n" + PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
+
     public static final String CREATE_LOG_METADATA =
             "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_LOG_TABLE + "\"(\n" +
-             // Pk columns
-            TENANT_ID + " VARCHAR ," +
-            QUERY_ID + " VARCHAR NOT NULL,\n" +
-            USER + " VARCHAR , \n" +
-            CLIENT_IP + " VARCHAR, \n" +
-            // Function metadata (will be null for argument row)
-            QUERY +  " VARCHAR, \n" +
-            EXPLAIN_PLAN + " VARCHAR, \n" +
-            // Argument metadata (will be null for function row)
-            START_TIME + " TIMESTAMP, \n" +
-            TOTAL_EXECUTION_TIME + " BIGINT, \n" +
-            NO_OF_RESULTS_ITERATED + " BIGINT, \n" +
-            QUERY_STATUS + " VARCHAR, \n" +
-            EXCEPTION_TRACE + " VARCHAR, \n" +
-            GLOBAL_SCAN_DETAILS + " VARCHAR, \n" +
-            BIND_PARAMETERS + " VARCHAR, \n" +
-            SCAN_METRICS_JSON + " VARCHAR, \n" +
-            " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (QUERY_ID))\n" +
-            HConstants.VERSIONS + "= " + MetaDataProtocol.DEFAULT_LOG_VERSIONS + ",\n" +
-            HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n"+
-            // Install split policy to prevent a tenant's metadata from being split across regions.
-            HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" + 
-            PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE+ ",\n" + 
-            HColumnDescriptor.TTL + "=" + MetaDataProtocol.DEFAULT_LOG_TTL+",\n"+
-            TableProperty.COLUMN_ENCODED_BYTES.toString()+" = 0";
-    
+            // Pk columns
+                    TENANT_ID + " VARCHAR ," + QUERY_ID + " VARCHAR NOT NULL,\n" + USER
+                    + " VARCHAR , \n" + CLIENT_IP + " VARCHAR, \n" +
+                    // Function metadata (will be null for argument row)
+                    QUERY + " VARCHAR, \n" + EXPLAIN_PLAN + " VARCHAR, \n" +
+                    // Argument metadata (will be null for function row)
+                    START_TIME + " TIMESTAMP, \n" + TOTAL_EXECUTION_TIME + " BIGINT, \n"
+                    + NO_OF_RESULTS_ITERATED + " BIGINT, \n" + QUERY_STATUS + " VARCHAR, \n"
+                    + EXCEPTION_TRACE + " VARCHAR, \n" + GLOBAL_SCAN_DETAILS + " VARCHAR, \n"
+                    + BIND_PARAMETERS + " VARCHAR, \n" + SCAN_METRICS_JSON + " VARCHAR, \n"
+                    + " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (QUERY_ID))\n"
+                    + HConstants.VERSIONS + "= " + MetaDataProtocol.DEFAULT_LOG_VERSIONS + ",\n"
+                    + HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n" +
+                    // Install split policy to prevent a tenant's metadata from being split across
+                    // regions.
+                    HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName()
+                    + "',\n" + PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE + ",\n"
+                    + HColumnDescriptor.TTL + "=" + MetaDataProtocol.DEFAULT_LOG_TTL + ",\n"
+                    + TableProperty.COLUMN_ENCODED_BYTES.toString() + " = 0";
+
     public static final byte[] OFFSET_FAMILY = "f_offset".getBytes();
     public static final byte[] OFFSET_COLUMN = "c_offset".getBytes();
     public static final String LAST_SCAN = "LAST_SCAN";
     public static final byte[] UPGRADE_MUTEX = "UPGRADE_MUTEX".getBytes();
     public static final String HASH_JOIN_CACHE_RETRIES = "hashjoin.client.retries.number";
     public static final int DEFAULT_HASH_JOIN_CACHE_RETRIES = 5;
-    
+
     // Links from parent to child views are stored in a separate table for scalability
     public static final String CREATE_CHILD_LINK_METADATA =
             "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CHILD_LINK_TABLE + "\"(\n" +
             // PK columns
-            TENANT_ID + " VARCHAR NULL," +
-            TABLE_SCHEM + " VARCHAR NULL," +
-            TABLE_NAME + " VARCHAR NOT NULL," +
-            COLUMN_NAME + " VARCHAR NULL," + 
-            COLUMN_FAMILY + " VARCHAR NULL," +
-            LINK_TYPE + " UNSIGNED_TINYINT,\n" +
-            "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ","
-            + TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" +
-            HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
-            HColumnDescriptor.KEEP_DELETED_CELLS + "="  + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n" +
-            PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
+                    TENANT_ID + " VARCHAR NULL," + TABLE_SCHEM + " VARCHAR NULL," + TABLE_NAME
+                    + " VARCHAR NOT NULL," + COLUMN_NAME + " VARCHAR NULL," + COLUMN_FAMILY
+                    + " VARCHAR NULL," + LINK_TYPE + " UNSIGNED_TINYINT,\n" + "CONSTRAINT "
+                    + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + TABLE_SCHEM + ","
+                    + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n"
+                    + HConstants.VERSIONS + "=%s,\n" + HColumnDescriptor.KEEP_DELETED_CELLS
+                    + "=%s,\n" 
+                    + PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/f8ce60c0/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index fbd7b7a..76d6267 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -307,7 +307,7 @@ public class PTableImpl implements PTable {
             table.getSequenceNumber(), table.getPKName(), basePTable.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(),
             table.getIndexes(), basePTable.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
             table.isWALDisabled(), basePTable.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
-            baseTableColumnCount, table.rowKeyOrderOptimizable(), basePTable.isTransactional(), table.getUpdateCacheFrequency(),
+            baseTableColumnCount, table.rowKeyOrderOptimizable(), basePTable.getTransactionProvider(), table.getUpdateCacheFrequency(),
             table.getIndexDisableTimestamp(), basePTable.isNamespaceMapped(), basePTable.getAutoPartitionSeqName(), basePTable.isAppendOnlySchema(),
             basePTable.getImmutableStorageScheme(), basePTable.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
     }


[14/50] [abbrv] phoenix git commit: PHOENIX-4605 Support running multiple transaction providers

Posted by td...@apache.org.
PHOENIX-4605 Support running multiple transaction providers


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/de83b8d5
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/de83b8d5
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/de83b8d5

Branch: refs/heads/system-catalog
Commit: de83b8d5d042098faa294e742525ab84175bb271
Parents: ca58fae
Author: James Taylor <jt...@salesforce.com>
Authored: Wed Apr 11 20:06:35 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Thu Apr 12 17:19:42 2018 -0700

----------------------------------------------------------------------
 .../phoenix/end2end/AlterTableWithViewsIT.java  |   6 +-
 .../ConnectionQueryServicesTestImpl.java        |  34 +-
 .../phoenix/tx/FlappingTransactionIT.java       |  11 +-
 .../phoenix/tx/ParameterizedTransactionIT.java  |  14 +-
 .../org/apache/phoenix/tx/TransactionIT.java    |  12 +
 .../org/apache/phoenix/tx/TxCheckpointIT.java   |   5 +-
 .../phoenix/cache/IndexMetaDataCache.java       |   5 +-
 .../apache/phoenix/compile/DeleteCompiler.java  |   3 +-
 .../apache/phoenix/compile/FromCompiler.java    |   2 +-
 .../apache/phoenix/compile/JoinCompiler.java    |   2 +-
 .../compile/TupleProjectionCompiler.java        |   4 +-
 .../apache/phoenix/compile/UnionCompiler.java   |   6 +-
 .../apache/phoenix/compile/UpsertCompiler.java  |   2 +-
 .../coprocessor/BaseScannerRegionObserver.java  |   4 +-
 .../coprocessor/MetaDataEndpointImpl.java       |  26 +-
 .../phoenix/coprocessor/MetaDataProtocol.java   |   2 +-
 .../coprocessor/MetaDataRegionObserver.java     |   4 +-
 .../PhoenixTransactionalProcessor.java          |  28 --
 .../coprocessor/ServerCachingEndpointImpl.java  |   4 +-
 .../TephraTransactionalProcessor.java           |  29 ++
 .../UngroupedAggregateRegionObserver.java       |  10 +-
 .../coprocessor/generated/PTableProtos.java     | 110 +++++-
 .../phoenix/exception/SQLExceptionCode.java     |   4 +
 .../apache/phoenix/execute/BaseQueryPlan.java   |   3 +
 .../apache/phoenix/execute/MutationState.java   |  74 ++--
 .../PhoenixTxIndexMutationGenerator.java        |  10 +-
 .../phoenix/expression/ExpressionType.java      | 119 +-----
 .../TransactionProviderNameFunction.java        |  81 ++++
 .../apache/phoenix/index/IndexMaintainer.java   |   4 +-
 .../index/IndexMetaDataCacheFactory.java        |   2 +-
 .../apache/phoenix/index/PhoenixIndexCodec.java |   1 -
 .../index/PhoenixIndexMetaDataBuilder.java      |   7 +-
 .../NonAggregateRegionScannerFactory.java       |   5 +-
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |  12 +-
 .../apache/phoenix/jdbc/PhoenixStatement.java   |   2 +-
 .../index/PhoenixIndexPartialBuildMapper.java   |   4 +-
 .../phoenix/query/ConnectionQueryServices.java  |   4 +
 .../query/ConnectionQueryServicesImpl.java      |  69 ++--
 .../query/ConnectionlessQueryServicesImpl.java  |   9 +-
 .../query/DelegateConnectionQueryServices.java  |   7 +-
 .../apache/phoenix/query/QueryConstants.java    |   2 +
 .../org/apache/phoenix/query/QueryServices.java |   1 +
 .../phoenix/query/QueryServicesOptions.java     |  10 +-
 .../apache/phoenix/schema/DelegateTable.java    |   8 +-
 .../apache/phoenix/schema/MetaDataClient.java   | 129 ++++--
 .../java/org/apache/phoenix/schema/PTable.java  |   2 +
 .../org/apache/phoenix/schema/PTableImpl.java   |  68 ++--
 .../apache/phoenix/schema/TableProperty.java    |  18 +
 .../transaction/OmidTransactionContext.java     |  57 +--
 .../transaction/OmidTransactionProvider.java    |  54 +--
 .../transaction/OmidTransactionTable.java       | 388 -------------------
 .../transaction/PhoenixTransactionClient.java   |  23 ++
 .../transaction/PhoenixTransactionContext.java  | 169 ++++----
 .../transaction/PhoenixTransactionProvider.java |  51 +++
 .../transaction/PhoenixTransactionService.java  |  24 ++
 .../transaction/PhoenixTransactionalTable.java  | 149 -------
 .../transaction/TephraTransactionContext.java   | 205 +++-------
 .../transaction/TephraTransactionProvider.java  | 161 +++++++-
 .../transaction/TephraTransactionTable.java     | 366 -----------------
 .../phoenix/transaction/TransactionFactory.java |  57 ++-
 .../transaction/TransactionProvider.java        |  36 --
 .../org/apache/phoenix/util/PhoenixRuntime.java |   3 +-
 .../java/org/apache/phoenix/util/ScanUtil.java  |  15 +
 .../apache/phoenix/util/TransactionUtil.java    |  49 ++-
 .../phoenix/execute/CorrelatePlanTest.java      |   5 +-
 .../execute/LiteralResultIteratorPlanTest.java  |   5 +-
 .../java/org/apache/phoenix/query/BaseTest.java |  26 +-
 .../phoenix/query/QueryServicesTestImpl.java    |  15 +-
 .../java/org/apache/phoenix/util/TestUtil.java  |   2 +-
 phoenix-protocol/src/main/PTable.proto          |   1 +
 70 files changed, 1176 insertions(+), 1663 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
index 6b57148..237a8d2 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
@@ -36,7 +36,7 @@ import java.util.Collection;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.coprocessor.PhoenixTransactionalProcessor;
+import org.apache.phoenix.coprocessor.TephraTransactionalProcessor;
 import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.query.QueryConstants;
@@ -748,7 +748,7 @@ public class AlterTableWithViewsIT extends ParallelStatsDisabledIT {
             PName tenantId = isMultiTenant ? PNameFactory.newName("tenant1") : null;
             PhoenixConnection phoenixConn = conn.unwrap(PhoenixConnection.class);
             HTableInterface htable = phoenixConn.getQueryServices().getTable(Bytes.toBytes(baseTableName));
-            assertFalse(htable.getTableDescriptor().getCoprocessors().contains(PhoenixTransactionalProcessor.class.getName()));
+            assertFalse(htable.getTableDescriptor().getCoprocessors().contains(TephraTransactionalProcessor.class.getName()));
             assertFalse(phoenixConn.getTable(new PTableKey(null, baseTableName)).isTransactional());
             assertFalse(viewConn.unwrap(PhoenixConnection.class).getTable(new PTableKey(tenantId, viewOfTable)).isTransactional());
             
@@ -757,7 +757,7 @@ public class AlterTableWithViewsIT extends ParallelStatsDisabledIT {
             // query the view to force the table cache to be updated
             viewConn.createStatement().execute("SELECT * FROM " + viewOfTable);
             htable = phoenixConn.getQueryServices().getTable(Bytes.toBytes(baseTableName));
-            assertTrue(htable.getTableDescriptor().getCoprocessors().contains(PhoenixTransactionalProcessor.class.getName()));
+            assertTrue(htable.getTableDescriptor().getCoprocessors().contains(TephraTransactionalProcessor.class.getName()));
             assertTrue(phoenixConn.getTable(new PTableKey(null, baseTableName)).isTransactional());
             assertTrue(viewConn.unwrap(PhoenixConnection.class).getTable(new PTableKey(tenantId, viewOfTable)).isTransactional());
         } 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
index a1ad1ad..3db93b0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.end2end;
 
 import static org.junit.Assert.assertEquals;
 
+import java.io.IOException;
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -29,7 +30,13 @@ import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
 import org.apache.phoenix.query.ConnectionQueryServicesImpl;
 import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.transaction.PhoenixTransactionClient;
+import org.apache.phoenix.transaction.PhoenixTransactionService;
+import org.apache.phoenix.transaction.TransactionFactory;
+import org.apache.phoenix.transaction.TransactionFactory.Provider;
 import org.apache.phoenix.util.SQLCloseables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Sets;
 
@@ -42,9 +49,11 @@ import com.google.common.collect.Sets;
  * @since 0.1
  */
 public class ConnectionQueryServicesTestImpl extends ConnectionQueryServicesImpl {
+    private static final Logger logger = LoggerFactory.getLogger(ConnectionQueryServicesTestImpl.class);
     protected int NUM_SLAVES_BASE = 1; // number of slaves for the cluster
     // Track open connections to free them on close as unit tests don't always do this.
     private Set<PhoenixConnection> connections = Sets.newHashSet();
+    private final PhoenixTransactionService[] txServices = new PhoenixTransactionService[TransactionFactory.Provider.values().length];
     
     public ConnectionQueryServicesTestImpl(QueryServices services, ConnectionInfo info, Properties props) throws SQLException {
         super(services, info, props);
@@ -68,12 +77,33 @@ public class ConnectionQueryServicesTestImpl extends ConnectionQueryServicesImpl
                 // Make copy to prevent ConcurrentModificationException (TODO: figure out why this is necessary)
                 connections = new ArrayList<>(this.connections);
                 this.connections = Sets.newHashSet();
+                
+                // shut down the tx client service if we created one to support transactions
+                for (PhoenixTransactionService service : txServices) {
+                    if (service != null) {
+                        try {
+                            service.close();
+                        } catch (IOException e) {
+                            logger.warn(e.getMessage(), e);
+                        }
+                    }
+                }
+
             }
             SQLCloseables.closeAll(connections);
-             long unfreedBytes = clearCache();
-             assertEquals("Found unfreed bytes in server-side cache", 0, unfreedBytes);
+            long unfreedBytes = clearCache();
+            assertEquals("Found unfreed bytes in server-side cache", 0, unfreedBytes);
         } finally {
             super.close();
         }
     }
+    
+    @Override
+    public synchronized PhoenixTransactionClient initTransactionClient(Provider provider) {
+        PhoenixTransactionService txService = txServices[provider.ordinal()];
+        if (txService == null) {
+            txService = txServices[provider.ordinal()] = provider.getTransactionProvider().getTransactionService(config, connectionInfo);
+        }
+        return super.initTransactionClient(provider);
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/it/java/org/apache/phoenix/tx/FlappingTransactionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/FlappingTransactionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/FlappingTransactionIT.java
index 200cf1c..0323c32 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/FlappingTransactionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/FlappingTransactionIT.java
@@ -43,7 +43,6 @@ import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.transaction.PhoenixTransactionContext;
-import org.apache.phoenix.transaction.PhoenixTransactionalTable;
 import org.apache.phoenix.transaction.TransactionFactory;
 import org.apache.phoenix.util.PropertiesUtil;
 import org.apache.phoenix.util.TestUtil;
@@ -225,9 +224,9 @@ public class FlappingTransactionIT extends ParallelStatsDisabledIT {
         }
 
         PhoenixTransactionContext txContext =
-              TransactionFactory.getTransactionProvider().getTransactionContext(pconn);
-        PhoenixTransactionalTable txTable =
-              TransactionFactory.getTransactionProvider().getTransactionalTable(txContext, htable);
+              TransactionFactory.getTransactionProvider(TransactionFactory.Provider.TEPHRA).getTransactionContext(pconn);
+        HTableInterface txTable =
+                txContext.getTransactionalTable(htable, false);
 
         txContext.begin();
 
@@ -277,9 +276,9 @@ public class FlappingTransactionIT extends ParallelStatsDisabledIT {
         
         // Repeat the same as above, but this time abort the transaction
         txContext =
-              TransactionFactory.getTransactionProvider().getTransactionContext(pconn);
+              TransactionFactory.getTransactionProvider(TransactionFactory.Provider.TEPHRA).getTransactionContext(pconn);
         txTable =
-              TransactionFactory.getTransactionProvider().getTransactionalTable(txContext, htable);
+              txContext.getTransactionalTable(htable, false);
 
         txContext.begin();
         

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
index fecfd9a..5421801 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/ParameterizedTransactionIT.java
@@ -16,8 +16,8 @@
  * limitations under the License.
  */
 package org.apache.phoenix.tx;
-import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
 import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -42,7 +42,7 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.coprocessor.PhoenixTransactionalProcessor;
+import org.apache.phoenix.coprocessor.TephraTransactionalProcessor;
 import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
 import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.jdbc.PhoenixConnection;
@@ -280,9 +280,9 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         conn.createStatement().execute("ALTER TABLE " + nonTxTableName + " SET TRANSACTIONAL=true");
         
         htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes( nonTxTableName));
-        assertTrue(htable.getTableDescriptor().getCoprocessors().contains(PhoenixTransactionalProcessor.class.getName()));
+        assertTrue(htable.getTableDescriptor().getCoprocessors().contains(TephraTransactionalProcessor.class.getName()));
         htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes(index));
-        assertTrue(htable.getTableDescriptor().getCoprocessors().contains(PhoenixTransactionalProcessor.class.getName()));
+        assertTrue(htable.getTableDescriptor().getCoprocessors().contains(TephraTransactionalProcessor.class.getName()));
 
         conn.createStatement().execute("UPSERT INTO " + nonTxTableName + " VALUES (4, 'c')");
         ResultSet rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ k FROM " + nonTxTableName + " WHERE v IS NULL");
@@ -357,7 +357,7 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         assertFalse(rs.next());
         
         htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("SYSTEM." + nonTxTableName));
-        assertFalse(htable.getTableDescriptor().getCoprocessors().contains(PhoenixTransactionalProcessor.class.getName()));
+        assertFalse(htable.getTableDescriptor().getCoprocessors().contains(TephraTransactionalProcessor.class.getName()));
         assertEquals(1,conn.unwrap(PhoenixConnection.class).getQueryServices().
                 getTableDescriptor(Bytes.toBytes("SYSTEM." + nonTxTableName)).
                 getFamily(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES).getMaxVersions());
@@ -375,7 +375,7 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         PTable table = pconn.getTable(new PTableKey(null, t1));
         HTableInterface htable = pconn.getQueryServices().getTable(Bytes.toBytes(t1));
         assertTrue(table.isTransactional());
-        assertTrue(htable.getTableDescriptor().getCoprocessors().contains(PhoenixTransactionalProcessor.class.getName()));
+        assertTrue(htable.getTableDescriptor().getCoprocessors().contains(TephraTransactionalProcessor.class.getName()));
         
         try {
             ddl = "ALTER TABLE " + t1 + " SET transactional=false";
@@ -409,7 +409,7 @@ public class ParameterizedTransactionIT extends ParallelStatsDisabledIT {
         table = pconn.getTable(new PTableKey(null, t1));
         htable = pconn.getQueryServices().getTable(Bytes.toBytes(t1));
         assertTrue(table.isTransactional());
-        assertTrue(htable.getTableDescriptor().getCoprocessors().contains(PhoenixTransactionalProcessor.class.getName()));
+        assertTrue(htable.getTableDescriptor().getCoprocessors().contains(TephraTransactionalProcessor.class.getName()));
     }
 
     @Test

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
index 9286c2e..f1344e0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
@@ -21,6 +21,7 @@ import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA;
 import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -44,6 +45,7 @@ import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.query.QueryServicesOptions;
 import org.apache.phoenix.schema.PTableKey;
 import org.apache.phoenix.transaction.PhoenixTransactionContext;
+import org.apache.phoenix.transaction.TransactionFactory;
 import org.apache.phoenix.util.EnvironmentEdgeManager;
 import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.PropertiesUtil;
@@ -129,6 +131,7 @@ public class TransactionIT  extends ParallelStatsDisabledIT {
             assertTrue(rs.next());
             assertEquals("Transactional table was not marked as transactional in JDBC API.",
                 "true", rs.getString(PhoenixDatabaseMetaData.TRANSACTIONAL));
+            assertEquals(TransactionFactory.Provider.TEPHRA.name(), rs.getString(PhoenixDatabaseMetaData.TRANSACTION_PROVIDER));
 
             String nonTransactTableName = generateUniqueName();
             Statement stmt2 = conn.createStatement();
@@ -139,6 +142,15 @@ public class TransactionIT  extends ParallelStatsDisabledIT {
             assertTrue(rs2.next());
             assertEquals("Non-transactional table was marked as transactional in JDBC API.",
                 "false", rs2.getString(PhoenixDatabaseMetaData.TRANSACTIONAL));
+            assertNull(rs2.getString(PhoenixDatabaseMetaData.TRANSACTION_PROVIDER));
+            
+            try {
+                stmt.execute("CREATE TABLE " + transactTableName + " (k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) " +
+                        "TRANSACTION_PROVIDER=foo");
+                fail();
+            } catch (SQLException e) {
+                assertEquals(SQLExceptionCode.UNKNOWN_TRANSACTION_PROVIDER.getErrorCode(), e.getErrorCode());
+            }
         }
     }
     

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java
index bf39dfe..42e2102 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java
@@ -38,6 +38,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.schema.PTableImpl;
 import org.apache.phoenix.transaction.PhoenixTransactionContext.PhoenixVisibilityLevel;
+import org.apache.phoenix.transaction.TransactionFactory;
 import org.apache.phoenix.util.PropertiesUtil;
 import org.apache.phoenix.util.SchemaUtil;
 import org.junit.Test;
@@ -265,7 +266,7 @@ public class TxCheckpointIT extends ParallelStatsDisabledIT {
 		ResultSet rs;
 		MutationState state = conn.unwrap(PhoenixConnection.class)
 				.getMutationState();
-		state.startTransaction();
+		state.startTransaction(TransactionFactory.Provider.TEPHRA);
 		long wp = state.getWritePointer();
 		conn.createStatement().execute(
 				"upsert into " + fullTableName + " select max(id)+1, 'a4', 'b4' from " + fullTableName + "");
@@ -329,7 +330,7 @@ public class TxCheckpointIT extends ParallelStatsDisabledIT {
 			conn.commit();
 
 	        MutationState state = conn.unwrap(PhoenixConnection.class).getMutationState();
-	        state.startTransaction();
+	        state.startTransaction(TransactionFactory.Provider.TEPHRA);
 	        long wp = state.getWritePointer();
 	        conn.createStatement().execute("delete from " + fullTableName + "1 where id1=fk1b AND fk1b=id1");
 	        assertEquals(PhoenixVisibilityLevel.SNAPSHOT, state.getVisibilityLevel());

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/cache/IndexMetaDataCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/cache/IndexMetaDataCache.java b/phoenix-core/src/main/java/org/apache/phoenix/cache/IndexMetaDataCache.java
index 17e6fb6..9f3dd59 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/IndexMetaDataCache.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/IndexMetaDataCache.java
@@ -23,12 +23,11 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 
-import org.apache.phoenix.hbase.index.util.VersionUtil;
 import org.apache.phoenix.index.IndexMaintainer;
 import org.apache.phoenix.transaction.PhoenixTransactionContext;
+import org.apache.phoenix.util.ScanUtil;
 
 public interface IndexMetaDataCache extends Closeable {
-    public static int UNKNOWN_CLIENT_VERSION = VersionUtil.encodeVersion(4, 4, 0);
     public static final IndexMetaDataCache EMPTY_INDEX_META_DATA_CACHE = new IndexMetaDataCache() {
 
         @Override
@@ -47,7 +46,7 @@ public interface IndexMetaDataCache extends Closeable {
         
         @Override
         public int getClientVersion() {
-            return UNKNOWN_CLIENT_VERSION;
+            return ScanUtil.UNKNOWN_CLIENT_VERSION;
         }
         
     };

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
index 7985314..5f9c76c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.phoenix.cache.ServerCacheClient;
 import org.apache.phoenix.cache.ServerCacheClient.ServerCache;
@@ -783,8 +782,8 @@ public class DeleteCompiler {
                     byte[] uuidValue = ServerCacheClient.generateId();
                     context.getScan().setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
                     context.getScan().setAttribute(PhoenixIndexCodec.INDEX_PROTO_MD, ptr.get());
-                    context.getScan().setAttribute(PhoenixIndexCodec.CLIENT_VERSION, Bytes.toBytes(MetaDataProtocol.PHOENIX_VERSION));
                     context.getScan().setAttribute(BaseScannerRegionObserver.TX_STATE, txState);
+                    ScanUtil.setClientVersion(context.getScan(), MetaDataProtocol.PHOENIX_VERSION);
                 }
                 ResultIterator iterator = aggPlan.iterator();
                 try {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
index b5293bb..3faada7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
@@ -821,7 +821,7 @@ public class FromCompiler {
             PTable t = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME, PTableType.SUBQUERY, null,
                     MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null,
                     Collections.<PTable> emptyList(), false, Collections.<PName> emptyList(), null, null, false, false,
-                    false, null, null, null, false, false, 0, 0L, SchemaUtil
+                    false, null, null, null, false, null, 0, 0L, SchemaUtil
                             .isNamespaceMappingEnabled(PTableType.SUBQUERY, connection.getQueryServices().getProps()), null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, true);
 
             String alias = subselectNode.getAlias();

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
index 88e8f50..824d933 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
@@ -1276,7 +1276,7 @@ public class JoinCompiler {
                 left.getBucketNum(), merged, left.getParentSchemaName(), left.getParentTableName(), left.getIndexes(),
                 left.isImmutableRows(), Collections.<PName> emptyList(), null, null, PTable.DEFAULT_DISABLE_WAL,
                 left.isMultiTenant(), left.getStoreNulls(), left.getViewType(), left.getViewIndexId(),
-                left.getIndexType(), left.rowKeyOrderOptimizable(), left.isTransactional(),
+                left.getIndexType(), left.rowKeyOrderOptimizable(), left.getTransactionProvider(),
                 left.getUpdateCacheFrequency(), left.getIndexDisableTimestamp(), left.isNamespaceMapped(), 
                 left.getAutoPartitionSeqName(), left.isAppendOnlySchema(), ONE_CELL_PER_COLUMN, NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, left.useStatsForParallelization());
     }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
index 6e52cd5..91be356 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
@@ -167,7 +167,7 @@ public class TupleProjectionCompiler {
                 table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), Collections.<PName> emptyList(),
                 table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(),
                 table.getViewIndexId(),
-                table.getIndexType(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(), 
+                table.getIndexType(), table.rowKeyOrderOptimizable(), table.getTransactionProvider(), table.getUpdateCacheFrequency(), 
                 table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
     }
     
@@ -198,7 +198,7 @@ public class TupleProjectionCompiler {
                 table.getBucketNum(), projectedColumns, null, null,
                 Collections.<PTable> emptyList(), table.isImmutableRows(), Collections.<PName> emptyList(), null, null,
                 table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(),
-                table.getViewIndexId(), null, table.rowKeyOrderOptimizable(), table.isTransactional(),
+                table.getViewIndexId(), null, table.rowKeyOrderOptimizable(), table.getTransactionProvider(),
                 table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), cqCounter, table.useStatsForParallelization());
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
index d5bfef8..9ca92f9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
@@ -35,12 +35,12 @@ import org.apache.phoenix.schema.PColumnImpl;
 import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PNameFactory;
 import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.PTableImpl;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.SortOrder;
 import org.apache.phoenix.schema.TableRef;
-import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
-import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 import org.apache.phoenix.schema.types.PDataType;
 import org.apache.phoenix.util.SchemaUtil;
 
@@ -100,7 +100,7 @@ public class UnionCompiler {
             UNION_SCHEMA_NAME, UNION_TABLE_NAME, PTableType.SUBQUERY, null,
             HConstants.LATEST_TIMESTAMP, scn == null ? HConstants.LATEST_TIMESTAMP : scn,
             null, null, projectedColumns, null, null, null, true, null, null, null, true,
-            true, true, null, null, null, false, false, 0, 0L,
+            true, true, null, null, null, false, null, 0, 0L,
             SchemaUtil.isNamespaceMappingEnabled(PTableType.SUBQUERY,
                 statement.getConnection().getQueryServices().getProps()), null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, true);
         TableRef tableRef = new TableRef(null, tempTable, 0, false);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
index 1a9a686..22119a3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
@@ -1020,8 +1020,8 @@ public class UpsertCompiler {
                 byte[] uuidValue = ServerCacheClient.generateId();
                 scan.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
                 scan.setAttribute(PhoenixIndexCodec.INDEX_PROTO_MD, ptr.get());
-                scan.setAttribute(PhoenixIndexCodec.CLIENT_VERSION, Bytes.toBytes(MetaDataProtocol.PHOENIX_VERSION));
                 scan.setAttribute(BaseScannerRegionObserver.TX_STATE, txState);
+                ScanUtil.setClientVersion(scan, MetaDataProtocol.PHOENIX_VERSION);
             }
             ResultIterator iterator = aggPlan.iterator();
             try {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index 110a4ed..60cc150 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.hbase.regionserver.ScanInfoUtil;
 import org.apache.hadoop.hbase.regionserver.ScannerContext;
 import org.apache.hadoop.hbase.regionserver.ScannerContextUtil;
 import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.StoreScanner;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.htrace.Span;
 import org.apache.htrace.Trace;
@@ -49,10 +48,8 @@ import org.apache.phoenix.hbase.index.covered.update.ColumnReference;
 import org.apache.phoenix.index.IndexMaintainer;
 import org.apache.phoenix.iterate.NonAggregateRegionScannerFactory;
 import org.apache.phoenix.iterate.RegionScannerFactory;
-import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.StaleRegionBoundaryCacheException;
 import org.apache.phoenix.schema.types.PUnsignedTinyint;
-import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.ServerUtil;
 import org.apache.phoenix.util.TransactionUtil;
@@ -110,6 +107,7 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
     public final static String QUALIFIER_ENCODING_SCHEME = "_QualifierEncodingScheme";
     public final static String IMMUTABLE_STORAGE_ENCODING_SCHEME = "_ImmutableStorageEncodingScheme";
     public final static String USE_ENCODED_COLUMN_QUALIFIER_LIST = "_UseEncodedColumnQualifierList";
+    public static final String CLIENT_VERSION = "_ClientVersion";
     
     public final static byte[] REPLAY_TABLE_AND_INDEX_WRITES = PUnsignedTinyint.INSTANCE.toBytes(1);
     public final static byte[] REPLAY_ONLY_INDEX_WRITES = PUnsignedTinyint.INSTANCE.toBytes(2);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 05ad959..a2d008b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -69,6 +69,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM_BYTE
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE_BYTES;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID_INDEX;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL_BYTES;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTION_PROVIDER_BYTES;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_BYTES;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY_BYTES;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION_BYTES;
@@ -228,6 +229,7 @@ import org.apache.phoenix.schema.types.PTinyint;
 import org.apache.phoenix.schema.types.PVarbinary;
 import org.apache.phoenix.schema.types.PVarchar;
 import org.apache.phoenix.trace.util.Tracing;
+import org.apache.phoenix.transaction.TransactionFactory;
 import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.EnvironmentEdgeManager;
@@ -295,6 +297,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
     private static final KeyValue BASE_COLUMN_COUNT_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.BASE_COLUMN_COUNT_BYTES);
     private static final KeyValue ROW_KEY_ORDER_OPTIMIZABLE_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, ROW_KEY_ORDER_OPTIMIZABLE_BYTES);
     private static final KeyValue TRANSACTIONAL_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, TRANSACTIONAL_BYTES);
+    private static final KeyValue TRANSACTION_PROVIDER_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, TRANSACTION_PROVIDER_BYTES);
     private static final KeyValue UPDATE_CACHE_FREQUENCY_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, UPDATE_CACHE_FREQUENCY_BYTES);
     private static final KeyValue IS_NAMESPACE_MAPPED_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY,
             TABLE_FAMILY_BYTES, IS_NAMESPACE_MAPPED_BYTES);
@@ -326,6 +329,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
             BASE_COLUMN_COUNT_KV,
             ROW_KEY_ORDER_OPTIMIZABLE_KV,
             TRANSACTIONAL_KV,
+            TRANSACTION_PROVIDER_KV,
             UPDATE_CACHE_FREQUENCY_KV,
             IS_NAMESPACE_MAPPED_KV,
             AUTO_PARTITION_SEQ_KV,
@@ -357,6 +361,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
     private static final int BASE_COLUMN_COUNT_INDEX = TABLE_KV_COLUMNS.indexOf(BASE_COLUMN_COUNT_KV);
     private static final int ROW_KEY_ORDER_OPTIMIZABLE_INDEX = TABLE_KV_COLUMNS.indexOf(ROW_KEY_ORDER_OPTIMIZABLE_KV);
     private static final int TRANSACTIONAL_INDEX = TABLE_KV_COLUMNS.indexOf(TRANSACTIONAL_KV);
+    private static final int TRANSACTION_PROVIDER_INDEX = TABLE_KV_COLUMNS.indexOf(TRANSACTION_PROVIDER_KV);
     private static final int UPDATE_CACHE_FREQUENCY_INDEX = TABLE_KV_COLUMNS.indexOf(UPDATE_CACHE_FREQUENCY_KV);
     private static final int INDEX_DISABLE_TIMESTAMP = TABLE_KV_COLUMNS.indexOf(INDEX_DISABLE_TIMESTAMP_KV);
     private static final int IS_NAMESPACE_MAPPED_INDEX = TABLE_KV_COLUMNS.indexOf(IS_NAMESPACE_MAPPED_KV);
@@ -961,7 +966,24 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
         Cell storeNullsKv = tableKeyValues[STORE_NULLS_INDEX];
         boolean storeNulls = storeNullsKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(storeNullsKv.getValueArray(), storeNullsKv.getValueOffset(), storeNullsKv.getValueLength()));
         Cell transactionalKv = tableKeyValues[TRANSACTIONAL_INDEX];
-        boolean transactional = transactionalKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(transactionalKv.getValueArray(), transactionalKv.getValueOffset(), transactionalKv.getValueLength()));
+        Cell transactionProviderKv = tableKeyValues[TRANSACTION_PROVIDER_INDEX];
+        TransactionFactory.Provider transactionProvider = null;
+        if (transactionProviderKv == null) {
+            if (transactionalKv != null && Boolean.TRUE.equals(
+                    PBoolean.INSTANCE.toObject(
+                            transactionalKv.getValueArray(), 
+                            transactionalKv.getValueOffset(), 
+                            transactionalKv.getValueLength()))) {
+                // For backward compat, prior to client setting TRANSACTION_PROVIDER
+                transactionProvider = TransactionFactory.Provider.TEPHRA;
+            }
+        } else {
+            transactionProvider = TransactionFactory.Provider.fromCode(
+                    PTinyint.INSTANCE.getCodec().decodeByte(
+                            transactionProviderKv.getValueArray(),
+                            transactionProviderKv.getValueOffset(), 
+                            SortOrder.getDefault()));
+        }
         Cell viewTypeKv = tableKeyValues[VIEW_TYPE_INDEX];
         ViewType viewType = viewTypeKv == null ? null : ViewType.fromSerializedValue(viewTypeKv.getValueArray()[viewTypeKv.getValueOffset()]);
         Cell viewIndexIdKv = tableKeyValues[VIEW_INDEX_ID_INDEX];
@@ -1044,7 +1066,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
         return PTableImpl.makePTable(tenantId, schemaName, tableName, tableType, indexState, timeStamp, tableSeqNum,
                 pkName, saltBucketNum, columns, parentSchemaName, parentTableName, indexes, isImmutableRows, physicalTables, defaultFamilyName,
                 viewStatement, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType,
-                rowKeyOrderOptimizable, transactional, updateCacheFrequency, baseColumnCount,
+                rowKeyOrderOptimizable, transactionProvider, updateCacheFrequency, baseColumnCount,
                 indexDisableTimestamp, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, storageScheme, encodingScheme, cqCounter, useStatsForParallelization);
     }
     

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index 4c4c96f..a71ce0c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -95,7 +95,7 @@ public abstract class MetaDataProtocol extends MetaDataService {
     // Since there's no upgrade code, keep the version the same as the previous version
     public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_12_0 = MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0;
     public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_13_0 = MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0;
-    public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0 = MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0;
+    public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0 = MIN_TABLE_TIMESTAMP + 28;
     // MIN_SYSTEM_TABLE_TIMESTAMP needs to be set to the max of all the MIN_SYSTEM_TABLE_TIMESTAMP_* constants
     public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0;
     

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
index 393a2f9..4968525 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
@@ -52,7 +52,6 @@ import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
@@ -83,6 +82,7 @@ import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.PropertiesUtil;
 import org.apache.phoenix.util.QueryUtil;
 import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.SchemaUtil;
 import org.apache.phoenix.util.ServerUtil;
 import org.apache.phoenix.util.UpgradeUtil;
@@ -496,7 +496,7 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
 									conn);
 							byte[] attribValue = ByteUtil.copyKeyBytesIfNecessary(indexMetaDataPtr);
 							dataTableScan.setAttribute(PhoenixIndexCodec.INDEX_PROTO_MD, attribValue);
-							dataTableScan.setAttribute(PhoenixIndexCodec.CLIENT_VERSION, Bytes.toBytes(MetaDataProtocol.PHOENIX_VERSION));
+							ScanUtil.setClientVersion(dataTableScan, MetaDataProtocol.PHOENIX_VERSION);
                             LOG.info("Starting to partially build indexes:" + indexesToPartiallyRebuild
                                     + " on data table:" + dataPTable.getName() + " with the earliest disable timestamp:"
                                     + earliestDisableTimestamp + " till "

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/PhoenixTransactionalProcessor.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/PhoenixTransactionalProcessor.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/PhoenixTransactionalProcessor.java
deleted file mode 100644
index 0c26ecc..0000000
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/PhoenixTransactionalProcessor.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.coprocessor;
-
-import org.apache.phoenix.transaction.TransactionFactory;
-
-public class PhoenixTransactionalProcessor extends DelegateRegionObserver {
-
-    public PhoenixTransactionalProcessor() {
-        super(TransactionFactory.getTransactionProvider().getTransactionContext().getCoprocessor());
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
index 448f61c..9d78659 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.phoenix.cache.GlobalCache;
-import org.apache.phoenix.cache.IndexMetaDataCache;
 import org.apache.phoenix.cache.TenantCache;
 import org.apache.phoenix.coprocessor.ServerCachingProtocol.ServerCacheFactory;
 import org.apache.phoenix.coprocessor.generated.ServerCachingProtos.AddServerCacheRequest;
@@ -37,6 +36,7 @@ import org.apache.phoenix.coprocessor.generated.ServerCachingProtos.ServerCachin
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 import org.apache.phoenix.protobuf.ProtobufUtil;
 import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.ServerUtil;
 
 import com.google.protobuf.RpcCallback;
@@ -75,7 +75,7 @@ public class ServerCachingEndpointImpl extends ServerCachingService implements C
           ServerCacheFactory cacheFactory = serverCacheFactoryClass.newInstance();
           tenantCache.addServerCache(new ImmutableBytesPtr(request.getCacheId().toByteArray()),
               cachePtr, txState, cacheFactory, request.hasHasProtoBufIndexMaintainer() && request.getHasProtoBufIndexMaintainer(),
-              request.hasClientVersion() ? request.getClientVersion() : IndexMetaDataCache.UNKNOWN_CLIENT_VERSION);
+              request.hasClientVersion() ? request.getClientVersion() : ScanUtil.UNKNOWN_CLIENT_VERSION);
         } catch (Throwable e) {
             ProtobufUtil.setControllerException(controller,
                 ServerUtil.createIOException("Error when adding cache: ", e));

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TephraTransactionalProcessor.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TephraTransactionalProcessor.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TephraTransactionalProcessor.java
new file mode 100644
index 0000000..a702bc4
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TephraTransactionalProcessor.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.coprocessor;
+
+import org.apache.tephra.hbase.coprocessor.TransactionProcessor;
+
+
+public class TephraTransactionalProcessor extends DelegateRegionObserver {
+
+    public TephraTransactionalProcessor() {
+        super(new TransactionProcessor());
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index de57772..31b512a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -277,7 +277,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
                m.setAttribute(BaseScannerRegionObserver.TX_STATE, txState);
            }
            if (clientVersionBytes != null) {
-               m.setAttribute(PhoenixIndexCodec.CLIENT_VERSION, clientVersionBytes);
+               m.setAttribute(BaseScannerRegionObserver.CLIENT_VERSION, clientVersionBytes);
            }
         }
     }
@@ -523,7 +523,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
             useIndexProto = false;
         }
 
-        byte[] clientVersionBytes = scan.getAttribute(PhoenixIndexCodec.CLIENT_VERSION);
+        byte[] clientVersionBytes = scan.getAttribute(BaseScannerRegionObserver.CLIENT_VERSION);
         boolean acquiredLock = false;
         boolean incrScanRefCount = false;
         final TenantCache tenantCache = GlobalCache.getTenantCache(env, ScanUtil.getTenantId(scan));
@@ -1023,7 +1023,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
             useProto = false;
             indexMetaData = scan.getAttribute(PhoenixIndexCodec.INDEX_MD);
         }
-        byte[] clientVersionBytes = scan.getAttribute(PhoenixIndexCodec.CLIENT_VERSION);
+        byte[] clientVersionBytes = scan.getAttribute(BaseScannerRegionObserver.CLIENT_VERSION);
         boolean hasMore;
         int rowCount = 0;
         try {
@@ -1048,7 +1048,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
                                     put.setAttribute(useProto ? PhoenixIndexCodec.INDEX_PROTO_MD : PhoenixIndexCodec.INDEX_MD, indexMetaData);
                                     put.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
                                     put.setAttribute(REPLAY_WRITES, REPLAY_ONLY_INDEX_WRITES);
-                                    put.setAttribute(PhoenixIndexCodec.CLIENT_VERSION, clientVersionBytes);
+                                    put.setAttribute(BaseScannerRegionObserver.CLIENT_VERSION, clientVersionBytes);
                                     mutations.add(put);
                                     // Since we're replaying existing mutations, it makes no sense to write them to the wal
                                     put.setDurability(Durability.SKIP_WAL);
@@ -1060,7 +1060,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
                                     del.setAttribute(useProto ? PhoenixIndexCodec.INDEX_PROTO_MD : PhoenixIndexCodec.INDEX_MD, indexMetaData);
                                     del.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
                                     del.setAttribute(REPLAY_WRITES, REPLAY_ONLY_INDEX_WRITES);
-                                    del.setAttribute(PhoenixIndexCodec.CLIENT_VERSION, clientVersionBytes);
+                                    del.setAttribute(BaseScannerRegionObserver.CLIENT_VERSION, clientVersionBytes);
                                     mutations.add(del);
                                     // Since we're replaying existing mutations, it makes no sense to write them to the wal
                                     del.setDurability(Durability.SKIP_WAL);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
index 278f301..8d500e8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
@@ -3520,6 +3520,16 @@ public final class PTableProtos {
      * <code>optional bool useStatsForParallelization = 37;</code>
      */
     boolean getUseStatsForParallelization();
+
+    // optional int32 transactionProvider = 38;
+    /**
+     * <code>optional int32 transactionProvider = 38;</code>
+     */
+    boolean hasTransactionProvider();
+    /**
+     * <code>optional int32 transactionProvider = 38;</code>
+     */
+    int getTransactionProvider();
   }
   /**
    * Protobuf type {@code PTable}
@@ -3771,6 +3781,11 @@ public final class PTableProtos {
               useStatsForParallelization_ = input.readBool();
               break;
             }
+            case 304: {
+              bitField1_ |= 0x00000001;
+              transactionProvider_ = input.readInt32();
+              break;
+            }
           }
         }
       } catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -3823,6 +3838,7 @@ public final class PTableProtos {
     }
 
     private int bitField0_;
+    private int bitField1_;
     // required bytes schemaNameBytes = 1;
     public static final int SCHEMANAMEBYTES_FIELD_NUMBER = 1;
     private com.google.protobuf.ByteString schemaNameBytes_;
@@ -4534,6 +4550,22 @@ public final class PTableProtos {
       return useStatsForParallelization_;
     }
 
+    // optional int32 transactionProvider = 38;
+    public static final int TRANSACTIONPROVIDER_FIELD_NUMBER = 38;
+    private int transactionProvider_;
+    /**
+     * <code>optional int32 transactionProvider = 38;</code>
+     */
+    public boolean hasTransactionProvider() {
+      return ((bitField1_ & 0x00000001) == 0x00000001);
+    }
+    /**
+     * <code>optional int32 transactionProvider = 38;</code>
+     */
+    public int getTransactionProvider() {
+      return transactionProvider_;
+    }
+
     private void initFields() {
       schemaNameBytes_ = com.google.protobuf.ByteString.EMPTY;
       tableNameBytes_ = com.google.protobuf.ByteString.EMPTY;
@@ -4571,6 +4603,7 @@ public final class PTableProtos {
       encodingScheme_ = com.google.protobuf.ByteString.EMPTY;
       encodedCQCounters_ = java.util.Collections.emptyList();
       useStatsForParallelization_ = false;
+      transactionProvider_ = 0;
     }
     private byte memoizedIsInitialized = -1;
     public final boolean isInitialized() {
@@ -4746,6 +4779,9 @@ public final class PTableProtos {
       if (((bitField0_ & 0x80000000) == 0x80000000)) {
         output.writeBool(37, useStatsForParallelization_);
       }
+      if (((bitField1_ & 0x00000001) == 0x00000001)) {
+        output.writeInt32(38, transactionProvider_);
+      }
       getUnknownFields().writeTo(output);
     }
 
@@ -4904,6 +4940,10 @@ public final class PTableProtos {
         size += com.google.protobuf.CodedOutputStream
           .computeBoolSize(37, useStatsForParallelization_);
       }
+      if (((bitField1_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(38, transactionProvider_);
+      }
       size += getUnknownFields().getSerializedSize();
       memoizedSerializedSize = size;
       return size;
@@ -5095,6 +5135,11 @@ public final class PTableProtos {
         result = result && (getUseStatsForParallelization()
             == other.getUseStatsForParallelization());
       }
+      result = result && (hasTransactionProvider() == other.hasTransactionProvider());
+      if (hasTransactionProvider()) {
+        result = result && (getTransactionProvider()
+            == other.getTransactionProvider());
+      }
       result = result &&
           getUnknownFields().equals(other.getUnknownFields());
       return result;
@@ -5252,6 +5297,10 @@ public final class PTableProtos {
         hash = (37 * hash) + USESTATSFORPARALLELIZATION_FIELD_NUMBER;
         hash = (53 * hash) + hashBoolean(getUseStatsForParallelization());
       }
+      if (hasTransactionProvider()) {
+        hash = (37 * hash) + TRANSACTIONPROVIDER_FIELD_NUMBER;
+        hash = (53 * hash) + getTransactionProvider();
+      }
       hash = (29 * hash) + getUnknownFields().hashCode();
       memoizedHashCode = hash;
       return hash;
@@ -5448,6 +5497,8 @@ public final class PTableProtos {
         }
         useStatsForParallelization_ = false;
         bitField1_ = (bitField1_ & ~0x00000008);
+        transactionProvider_ = 0;
+        bitField1_ = (bitField1_ & ~0x00000010);
         return this;
       }
 
@@ -5477,6 +5528,7 @@ public final class PTableProtos {
         int from_bitField0_ = bitField0_;
         int from_bitField1_ = bitField1_;
         int to_bitField0_ = 0;
+        int to_bitField1_ = 0;
         if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
           to_bitField0_ |= 0x00000001;
         }
@@ -5637,7 +5689,12 @@ public final class PTableProtos {
           to_bitField0_ |= 0x80000000;
         }
         result.useStatsForParallelization_ = useStatsForParallelization_;
+        if (((from_bitField1_ & 0x00000010) == 0x00000010)) {
+          to_bitField1_ |= 0x00000001;
+        }
+        result.transactionProvider_ = transactionProvider_;
         result.bitField0_ = to_bitField0_;
+        result.bitField1_ = to_bitField1_;
         onBuilt();
         return result;
       }
@@ -5841,6 +5898,9 @@ public final class PTableProtos {
         if (other.hasUseStatsForParallelization()) {
           setUseStatsForParallelization(other.getUseStatsForParallelization());
         }
+        if (other.hasTransactionProvider()) {
+          setTransactionProvider(other.getTransactionProvider());
+        }
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
@@ -7920,6 +7980,39 @@ public final class PTableProtos {
         return this;
       }
 
+      // optional int32 transactionProvider = 38;
+      private int transactionProvider_ ;
+      /**
+       * <code>optional int32 transactionProvider = 38;</code>
+       */
+      public boolean hasTransactionProvider() {
+        return ((bitField1_ & 0x00000010) == 0x00000010);
+      }
+      /**
+       * <code>optional int32 transactionProvider = 38;</code>
+       */
+      public int getTransactionProvider() {
+        return transactionProvider_;
+      }
+      /**
+       * <code>optional int32 transactionProvider = 38;</code>
+       */
+      public Builder setTransactionProvider(int value) {
+        bitField1_ |= 0x00000010;
+        transactionProvider_ = value;
+        onChanged();
+        return this;
+      }
+      /**
+       * <code>optional int32 transactionProvider = 38;</code>
+       */
+      public Builder clearTransactionProvider() {
+        bitField1_ = (bitField1_ & ~0x00000010);
+        transactionProvider_ = 0;
+        onChanged();
+        return this;
+      }
+
       // @@protoc_insertion_point(builder_scope:PTable)
     }
 
@@ -8587,7 +8680,7 @@ public final class PTableProtos {
       "\"\232\001\n\013PTableStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030",
       "\002 \003(\014\022\033\n\023guidePostsByteCount\030\003 \001(\003\022\025\n\rke" +
       "yBytesCount\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001" +
-      "(\005\022!\n\013pGuidePosts\030\006 \001(\0132\014.PGuidePosts\"\220\007" +
+      "(\005\022!\n\013pGuidePosts\030\006 \001(\0132\014.PGuidePosts\"\255\007" +
       "\n\006PTable\022\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tab" +
       "leNameBytes\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.P" +
       "TableType\022\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenc" +
@@ -8610,12 +8703,13 @@ public final class PTableProtos {
       "\n\017parentNameBytes\030! \001(\014\022\025\n\rstorageScheme" +
       "\030\" \001(\014\022\026\n\016encodingScheme\030# \001(\014\022,\n\021encode" +
       "dCQCounters\030$ \003(\0132\021.EncodedCQCounter\022\"\n\032" +
-      "useStatsForParallelization\030% \001(\010\"6\n\020Enco" +
-      "dedCQCounter\022\021\n\tcolFamily\030\001 \002(\t\022\017\n\007count" +
-      "er\030\002 \002(\005*A\n\nPTableType\022\n\n\006SYSTEM\020\000\022\010\n\004US" +
-      "ER\020\001\022\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(o" +
-      "rg.apache.phoenix.coprocessor.generatedB",
-      "\014PTableProtosH\001\210\001\001\240\001\001"
+      "useStatsForParallelization\030% \001(\010\022\033\n\023tran" +
+      "sactionProvider\030& \001(\005\"6\n\020EncodedCQCounte" +
+      "r\022\021\n\tcolFamily\030\001 \002(\t\022\017\n\007counter\030\002 \002(\005*A\n" +
+      "\nPTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIE" +
+      "W\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.p",
+      "hoenix.coprocessor.generatedB\014PTableProt" +
+      "osH\001\210\001\001\240\001\001"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
       new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -8639,7 +8733,7 @@ public final class PTableProtos {
           internal_static_PTable_fieldAccessorTable = new
             com.google.protobuf.GeneratedMessage.FieldAccessorTable(
               internal_static_PTable_descriptor,
-              new java.lang.String[] { "SchemaNameBytes", "TableNameBytes", "TableType", "IndexState", "SequenceNumber", "TimeStamp", "PkNameBytes", "BucketNum", "Columns", "Indexes", "IsImmutableRows", "DataTableNameBytes", "DefaultFamilyName", "DisableWAL", "MultiTenant", "ViewType", "ViewStatement", "PhysicalNames", "TenantId", "ViewIndexId", "IndexType", "StatsTimeStamp", "StoreNulls", "BaseColumnCount", "RowKeyOrderOptimizable", "Transactional", "UpdateCacheFrequency", "IndexDisableTimestamp", "IsNamespaceMapped", "AutoParititonSeqName", "IsAppendOnlySchema", "ParentNameBytes", "StorageScheme", "EncodingScheme", "EncodedCQCounters", "UseStatsForParallelization", });
+              new java.lang.String[] { "SchemaNameBytes", "TableNameBytes", "TableType", "IndexState", "SequenceNumber", "TimeStamp", "PkNameBytes", "BucketNum", "Columns", "Indexes", "IsImmutableRows", "DataTableNameBytes", "DefaultFamilyName", "DisableWAL", "MultiTenant", "ViewType", "ViewStatement", "PhysicalNames", "TenantId", "ViewIndexId", "IndexType", "StatsTimeStamp", "StoreNulls", "BaseColumnCount", "RowKeyOrderOptimizable", "Transactional", "UpdateCacheFrequency", "IndexDisableTimestamp", "IsNamespaceMapped", "AutoParititonSeqName", "IsAppendOnlySchema", "ParentNameBytes", "StorageScheme", "EncodingScheme", "EncodedCQCounters", "UseStatsForParallelization", "TransactionProvider", });
           internal_static_EncodedCQCounter_descriptor =
             getDescriptor().getMessageTypes().get(3);
           internal_static_EncodedCQCounter_fieldAccessorTable = new

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index 9cbc67e..e9e209b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -294,6 +294,10 @@ public enum SQLExceptionCode {
     SEQUENCE_NOT_CASTABLE_TO_AUTO_PARTITION_ID_COLUMN(1086, "44A17", "Sequence Value not castable to auto-partition id column"),
     CANNOT_COERCE_AUTO_PARTITION_ID(1087, "44A18", "Auto-partition id cannot be coerced"),
     CANNOT_CREATE_INDEX_ON_MUTABLE_TABLE_WITH_ROWTIMESTAMP(1088, "44A19", "Cannot create an index on a mutable table that has a ROW_TIMESTAMP column."),
+    UNKNOWN_TRANSACTION_PROVIDER(1089,"44A20", "Unknown TRANSACTION_PROVIDER: "),
+    CANNOT_START_TXN_IF_TXN_DISABLED(1091, "44A22", "Cannot start transaction if transactions are disabled."),
+    CANNOT_MIX_TXN_PROVIDERS(1092, "44A23", "Cannot mix transaction providers: "),
+    CANNOT_ALTER_TABLE_FROM_NON_TXN_TO_TXNL(1093, "44A24", "Cannot alter table from non transactional to transactional for "),
 
     /** Sequence related */
     SEQUENCE_ALREADY_EXIST(1200, "42Z00", "Sequence already exists.", new Factory() {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
index b152030..8a8c822 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
@@ -47,6 +47,7 @@ import org.apache.phoenix.compile.ScanRanges;
 import org.apache.phoenix.compile.StatementContext;
 import org.apache.phoenix.compile.WhereCompiler;
 import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
+import org.apache.phoenix.coprocessor.MetaDataProtocol;
 import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.expression.ProjectedColumnExpression;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
@@ -252,6 +253,8 @@ public abstract class BaseQueryPlan implements QueryPlan {
             return newIterator(scanGrouper, scan, caches);
         }
         
+        ScanUtil.setClientVersion(scan, MetaDataProtocol.PHOENIX_VERSION);
+        
         // Set miscellaneous scan attributes. This is the last chance to set them before we
         // clone the scan for each parallelized chunk.
         TableRef tableRef = context.getCurrentTable();

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
index 839194a..18f4fea 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -72,6 +72,8 @@ import org.apache.phoenix.monitoring.MutationMetricQueue.MutationMetric;
 import org.apache.phoenix.monitoring.MutationMetricQueue.NoOpMutationMetricsQueue;
 import org.apache.phoenix.monitoring.ReadMetricQueue;
 import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
 import org.apache.phoenix.schema.IllegalDataException;
 import org.apache.phoenix.schema.MetaDataClient;
 import org.apache.phoenix.schema.PColumn;
@@ -91,8 +93,8 @@ import org.apache.phoenix.schema.types.PLong;
 import org.apache.phoenix.trace.util.Tracing;
 import org.apache.phoenix.transaction.PhoenixTransactionContext;
 import org.apache.phoenix.transaction.PhoenixTransactionContext.PhoenixVisibilityLevel;
-import org.apache.phoenix.transaction.PhoenixTransactionalTable;
 import org.apache.phoenix.transaction.TransactionFactory;
+import org.apache.phoenix.transaction.TransactionFactory.Provider;
 import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.KeyValueUtil;
@@ -139,7 +141,7 @@ public class MutationState implements SQLCloseable {
     private boolean isExternalTxContext = false;
     private Map<TableRef, MultiRowMutationState> txMutations = Collections.emptyMap();
 
-    final PhoenixTransactionContext phoenixTransactionContext;
+    private PhoenixTransactionContext phoenixTransactionContext = PhoenixTransactionContext.NULL_CONTEXT;
 
     private final MutationMetricQueue mutationMetricQueue;
     private ReadMetricQueue readMetricQueue;
@@ -181,17 +183,13 @@ public class MutationState implements SQLCloseable {
         boolean isMetricsEnabled = connection.isRequestLevelMetricsEnabled();
         this.mutationMetricQueue = isMetricsEnabled ? new MutationMetricQueue()
                 : NoOpMutationMetricsQueue.NO_OP_MUTATION_METRICS_QUEUE;
-        if (!subTask) {
-            if (txContext == null) {
-                phoenixTransactionContext = TransactionFactory.getTransactionProvider().getTransactionContext(connection);
-            } else {
-                isExternalTxContext = true;
-                phoenixTransactionContext = TransactionFactory.getTransactionProvider().getTransactionContext(txContext, connection, subTask);
-            }
-        } else {
+        if (subTask) {
             // this code path is only used while running child scans, we can't pass the txContext to child scans
             // as it is not thread safe, so we use the tx member variable
-            phoenixTransactionContext = TransactionFactory.getTransactionProvider().getTransactionContext(txContext, connection, subTask);
+            phoenixTransactionContext = txContext.newTransactionContext(txContext, subTask);
+        } else if (txContext != null) {
+            isExternalTxContext = true;
+            phoenixTransactionContext = txContext.newTransactionContext(txContext, subTask);
         }
     }
 
@@ -234,7 +232,7 @@ public class MutationState implements SQLCloseable {
     public void commitDDLFence(PTable dataTable) throws SQLException {
         if (dataTable.isTransactional()) {
             try {
-                phoenixTransactionContext.commitDDLFence(dataTable, logger);
+                phoenixTransactionContext.commitDDLFence(dataTable);
             } finally {
                 // The client expects a transaction to be in progress on the txContext while the
                 // VisibilityFence.prepareWait() starts a new tx and finishes/aborts it. After it's
@@ -300,14 +298,12 @@ public class MutationState implements SQLCloseable {
     // Though MutationState is not thread safe in general, this method should be because it may
     // be called by TableResultIterator in a multi-threaded manner. Since we do not want to expose
     // the Transaction outside of MutationState, this seems reasonable, as the member variables
-    // would not change as these threads are running.
+    // would not change as these threads are running. We also clone mutationState to ensure that
+    // the transaction context won't change due to a commit when auto commit is true.
     public HTableInterface getHTable(PTable table) throws SQLException {
         HTableInterface htable = this.getConnection().getQueryServices().getTable(table.getPhysicalName().getBytes());
         if (table.isTransactional() && phoenixTransactionContext.isTransactionRunning()) {
-            PhoenixTransactionalTable phoenixTransactionTable = TransactionUtil.getPhoenixTransactionTable(phoenixTransactionContext, htable, table);
-            // Using cloned mutationState as we may have started a new transaction already
-            // if auto commit is true and we need to use the original one here.
-            htable = phoenixTransactionTable;
+            htable = phoenixTransactionContext.getTransactionalTable(htable, table.isImmutableRows());
         }
         return htable;
     }
@@ -334,13 +330,32 @@ public class MutationState implements SQLCloseable {
         return phoenixTransactionContext.getVisibilityLevel();
     }
 
-    public boolean startTransaction() throws SQLException {
+    public boolean startTransaction(Provider provider) throws SQLException {
+        if (provider == null) {
+            return false;
+        }
+        if (!connection.getQueryServices().getProps().getBoolean(
+                QueryServices.TRANSACTIONS_ENABLED,
+                QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED)) {
+            throw new SQLExceptionInfo.Builder(
+                    SQLExceptionCode.CANNOT_START_TXN_IF_TXN_DISABLED)
+                    .build().buildException();
+        }
         if (connection.getSCN() != null) {
             throw new SQLExceptionInfo.Builder(
                     SQLExceptionCode.CANNOT_START_TRANSACTION_WITH_SCN_SET)
                     .build().buildException();
         }
 
+        if (phoenixTransactionContext == PhoenixTransactionContext.NULL_CONTEXT) {
+            phoenixTransactionContext = provider.getTransactionProvider().getTransactionContext(connection);
+        } else {
+            if (provider != phoenixTransactionContext.getProvider()) {
+                throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_MIX_TXN_PROVIDERS)
+                        .setMessage(phoenixTransactionContext.getProvider().name() + " and " + provider.name())
+                        .build().buildException();
+            }
+        }
         if (!isTransactionStarted()) {
             // Clear any transactional state in case transaction was ended outside
             // of Phoenix so we don't carry the old transaction state forward. We
@@ -1069,6 +1084,7 @@ public class MutationState implements SQLCloseable {
                         if (table.isTransactional()) {
                             // Track tables to which we've sent uncommitted data
                             uncommittedPhysicalNames.add(table.getPhysicalName().getString());
+                            phoenixTransactionContext.markDMLFence(table);
 
                             // If we have indexes, wrap the HTable in a delegate HTable that
                             // will attach the necessary index meta data in the event of a
@@ -1077,7 +1093,7 @@ public class MutationState implements SQLCloseable {
                                 hTable = new MetaDataAwareHTable(hTable, origTableRef);
                             }
 
-                            hTable = TransactionUtil.getPhoenixTransactionTable(phoenixTransactionContext, hTable, table);
+                            hTable = phoenixTransactionContext.getTransactionalTable(hTable, table.isImmutableRows());
                         }
                         
                         numMutations = mutationList.size();
@@ -1223,10 +1239,6 @@ public class MutationState implements SQLCloseable {
         return phoenixTransactionContext.encodeTransaction();
     }
 
-    public static PhoenixTransactionContext decodeTransaction(byte[] txnBytes) throws IOException {
-        return TransactionFactory.getTransactionProvider().getTransactionContext(txnBytes);
-    }
-
     private ServerCache setMetaDataOnMutations(TableRef tableRef, List<? extends Mutation> mutations,
             ImmutableBytesWritable indexMetaDataPtr) throws SQLException {
         PTable table = tableRef.getTable();
@@ -1269,7 +1281,7 @@ public class MutationState implements SQLCloseable {
             mutation.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
             if (attribValue != null) {
                 mutation.setAttribute(PhoenixIndexCodec.INDEX_PROTO_MD, attribValue);
-                mutation.setAttribute(PhoenixIndexCodec.CLIENT_VERSION, Bytes.toBytes(MetaDataProtocol.PHOENIX_VERSION));
+                mutation.setAttribute(BaseScannerRegionObserver.CLIENT_VERSION, Bytes.toBytes(MetaDataProtocol.PHOENIX_VERSION));
                 if (txState.length > 0) {
                     mutation.setAttribute(BaseScannerRegionObserver.TX_STATE, txState);
                 }
@@ -1301,7 +1313,7 @@ public class MutationState implements SQLCloseable {
         numRows = 0;
         estimatedSize = 0;
         this.mutations.clear();
-        resetTransactionalState();
+        phoenixTransactionContext = PhoenixTransactionContext.NULL_CONTEXT;
     }
 
     private void resetTransactionalState() {
@@ -1365,13 +1377,18 @@ public class MutationState implements SQLCloseable {
                         }
                     }
                 } finally {
+                    TransactionFactory.Provider provider = phoenixTransactionContext.getProvider();
                     try {
                         resetState();
                     } finally {
                         if (retryCommit) {
-                            startTransaction();
+                            startTransaction(provider);
                             // Add back read fences
                             Set<TableRef> txTableRefs = txMutations.keySet();
+                            for (TableRef tableRef : txTableRefs) {
+                                PTable dataTable = tableRef.getTable();
+                                phoenixTransactionContext.markDMLFence(dataTable);
+                            }
                             try {
                                 // Only retry if an index was added
                                 retryCommit = shouldResubmitTransaction(txTableRefs);
@@ -1476,9 +1493,12 @@ public class MutationState implements SQLCloseable {
             List<TableRef> strippedAliases = Lists.newArrayListWithExpectedSize(mutations.keySet().size());
             while (filteredTableRefs.hasNext()) {
                 TableRef tableRef = filteredTableRefs.next();
+                // REVIEW: unclear if we need this given we start transactions when resolving a table
+                if (tableRef.getTable().isTransactional()) {
+                    startTransaction(tableRef.getTable().getTransactionProvider());
+                }
                 strippedAliases.add(new TableRef(null, tableRef.getTable(), tableRef.getTimeStamp(), tableRef.getLowerBoundTimeStamp(), tableRef.hasDynamicCols()));
             }
-            startTransaction();
             send(strippedAliases.iterator());
             return true;
         }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/execute/PhoenixTxIndexMutationGenerator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/PhoenixTxIndexMutationGenerator.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/PhoenixTxIndexMutationGenerator.java
index 7d6154e..6348d6d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/PhoenixTxIndexMutationGenerator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/PhoenixTxIndexMutationGenerator.java
@@ -31,8 +31,6 @@ import java.util.ListIterator;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellScanner;
@@ -43,6 +41,7 @@ import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
@@ -64,8 +63,6 @@ import org.apache.phoenix.query.KeyRange;
 import org.apache.phoenix.schema.types.PVarbinary;
 import org.apache.phoenix.transaction.PhoenixTransactionContext;
 import org.apache.phoenix.transaction.PhoenixTransactionContext.PhoenixVisibilityLevel;
-import org.apache.phoenix.transaction.PhoenixTransactionalTable;
-import org.apache.phoenix.transaction.TransactionFactory;
 import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.SchemaUtil;
 
@@ -76,9 +73,6 @@ import com.google.common.primitives.Longs;
 
 
 public class PhoenixTxIndexMutationGenerator {
-
-    private static final Log LOG = LogFactory.getLog(PhoenixTxIndexMutationGenerator.class);
-
     private final PhoenixIndexCodec codec;
     private final PhoenixIndexMetaData indexMetaData;
 
@@ -181,7 +175,7 @@ public class PhoenixTxIndexMutationGenerator {
             scan.addColumn(indexMaintainers.get(0).getDataEmptyKeyValueCF(), emptyKeyValueQualifier);
             ScanRanges scanRanges = ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, KeyRange.EVERYTHING_RANGE, null, true, -1);
             scanRanges.initializeScan(scan);
-            PhoenixTransactionalTable txTable = TransactionFactory.getTransactionProvider().getTransactionalTable(indexMetaData.getTransactionContext(), htable);
+            Table txTable = indexMetaData.getTransactionContext().getTransactionalTable(htable, isImmutable);
             // For rollback, we need to see all versions, including
             // the last committed version as there may be multiple
             // checkpointed versions.


[04/50] [abbrv] phoenix git commit: PHOENIX-2715 Query Log (addendum)

Posted by td...@apache.org.
PHOENIX-2715 Query Log (addendum)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ed636616
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ed636616
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ed636616

Branch: refs/heads/system-catalog
Commit: ed636616138cbb8db4b71c9cf8a1b2fa7b503aa0
Parents: 0bc0368
Author: Ankit Singhal <an...@gmail.com>
Authored: Tue Apr 10 15:41:51 2018 +0530
Committer: Ankit Singhal <an...@gmail.com>
Committed: Tue Apr 10 15:41:51 2018 +0530

----------------------------------------------------------------------
 .../MigrateSystemTablesToSystemNamespaceIT.java | 40 +++++++++++---------
 1 file changed, 22 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed636616/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
index c36cdd3..7cb5857 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
@@ -16,6 +16,26 @@
  */
 package org.apache.phoenix.end2end;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
@@ -38,31 +58,15 @@ import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-
-import static org.junit.Assert.*;
-
 @Category(NeedsOwnMiniClusterTest.class)
 public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest {
 
     private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
             "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
-            "SYSTEM.MUTEX"));
+            "SYSTEM.MUTEX","SYSTEM.LOG"));
     private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
             Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
-                    "SYSTEM:MUTEX"));
+                    "SYSTEM:MUTEX","SYSTEM:LOG"));
     private static final String SCHEMA_NAME = "MIGRATETEST";
     private static final String TABLE_NAME =
             SCHEMA_NAME + "." + MigrateSystemTablesToSystemNamespaceIT.class.getSimpleName().toUpperCase();


[19/50] [abbrv] phoenix git commit: PHOENIX-4579 Add a config to conditionally create Phoenix meta tables on first client connection (Chinmay Kulkarni)

Posted by td...@apache.org.
PHOENIX-4579 Add a config to conditionally create Phoenix meta tables on first client connection (Chinmay Kulkarni)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/87fdda8b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/87fdda8b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/87fdda8b

Branch: refs/heads/system-catalog
Commit: 87fdda8b180438e57d0e2f6082e5a9e988220245
Parents: 7b2c7e1
Author: James Taylor <jt...@salesforce.com>
Authored: Fri Apr 13 10:30:30 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Apr 13 10:30:30 2018 -0700

----------------------------------------------------------------------
 .../phoenix/end2end/AppendOnlySchemaIT.java     |   2 +-
 .../MigrateSystemTablesToSystemNamespaceIT.java |   9 +-
 .../SystemCatalogCreationOnConnectionIT.java    | 626 ++++++++++++++++
 .../coprocessor/MetaDataEndpointImpl.java       |  21 +
 .../phoenix/coprocessor/MetaDataProtocol.java   |   4 +
 .../coprocessor/generated/MetaDataProtos.java   | 183 +++--
 .../exception/UpgradeRequiredException.java     |  13 +-
 .../phoenix/query/ConnectionQueryServices.java  |   2 +-
 .../query/ConnectionQueryServicesImpl.java      | 750 +++++++++++--------
 .../query/ConnectionlessQueryServicesImpl.java  |   2 +-
 .../query/DelegateConnectionQueryServices.java  |   4 +-
 .../apache/phoenix/schema/MetaDataClient.java   |   3 +-
 .../query/ConnectionQueryServicesImplTest.java  |   7 +-
 phoenix-protocol/src/main/MetaDataService.proto |   1 +
 14 files changed, 1236 insertions(+), 391 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AppendOnlySchemaIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AppendOnlySchemaIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AppendOnlySchemaIT.java
index 7ed64ff..d601beb 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AppendOnlySchemaIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AppendOnlySchemaIT.java
@@ -114,7 +114,7 @@ public class AppendOnlySchemaIT extends ParallelStatsDisabledIT {
             // verify no create table rpcs
             verify(connectionQueryServices, never()).createTable(anyListOf(Mutation.class),
                 any(byte[].class), any(PTableType.class), anyMap(), anyList(), any(byte[][].class),
-                eq(false), eq(false));
+                eq(false), eq(false), eq(false));
             reset(connectionQueryServices);
             
             // execute alter table ddl that adds the same column

http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
index 7cb5857..627e453 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
@@ -376,8 +376,10 @@ public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest {
         while(rs.next()) {
 
             if(rs.getString("IS_NAMESPACE_MAPPED") == null) {
+                // Check that entry for SYSTEM namespace exists in SYSCAT
                 systemSchemaExists = rs.getString("TABLE_SCHEM").equals(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME) ? true : systemSchemaExists;
             } else if (rs.getString("COLUMN_NAME") == null) {
+                // Found the intial entry for a table in SYSCAT
                 String schemaName = rs.getString("TABLE_SCHEM");
                 String tableName = rs.getString("TABLE_NAME");
 
@@ -395,12 +397,11 @@ public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest {
             }
         }
 
-        if(!systemSchemaExists) {
-            fail(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME + " entry doesn't exist in SYSTEM.CATALOG table.");
-        }
-
         // The set will contain SYSMUTEX table since that table is not exposed in SYSCAT
         if (systemTablesMapped) {
+            if (!systemSchemaExists) {
+                fail(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME + " entry doesn't exist in SYSTEM.CATALOG table.");
+            }
             assertTrue(namespaceMappedSystemTablesSet.size() == 1);
         } else {
             assertTrue(systemTablesSet.size() == 1);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
new file mode 100644
index 0000000..689eb20
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
@@ -0,0 +1,626 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.phoenix.coprocessor.MetaDataProtocol;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.exception.UpgradeRequiredException;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver;
+import org.apache.phoenix.jdbc.PhoenixTestDriver;
+import org.apache.phoenix.query.*;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.UpgradeUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.concurrent.TimeoutException;
+
+import static org.junit.Assert.*;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class SystemCatalogCreationOnConnectionIT {
+    private HBaseTestingUtility testUtil = null;
+    private Set<String> hbaseTables;
+    private static boolean setOldTimestampToInduceUpgrade = false;
+    private static int countUpgradeAttempts;
+    // This flag is used to figure out if the SYSCAT schema was actually upgraded or not, based on the timestamp of SYSCAT
+    // (different from an upgrade attempt)
+    private static int actualSysCatUpgrades;
+    private static final String PHOENIX_NAMESPACE_MAPPED_SYSTEM_CATALOG = "SYSTEM:CATALOG";
+    private static final String PHOENIX_SYSTEM_CATALOG = "SYSTEM.CATALOG";
+    private static final String EXECUTE_UPGRADE_COMMAND = "EXECUTE UPGRADE";
+    private static final String MODIFIED_MAX_VERSIONS ="5";
+
+    private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
+      "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
+      "SYSTEM.MUTEX", "SYSTEM.LOG"));
+
+    private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
+      Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
+        "SYSTEM:MUTEX", "SYSTEM:LOG"));
+
+    private static class PhoenixSysCatCreationServices extends ConnectionQueryServicesImpl {
+
+        public PhoenixSysCatCreationServices(QueryServices services, PhoenixEmbeddedDriver.ConnectionInfo connectionInfo, Properties info) {
+            super(services, connectionInfo, info);
+        }
+
+        @Override
+        protected void setUpgradeRequired() {
+            super.setUpgradeRequired();
+            countUpgradeAttempts++;
+        }
+
+        @Override
+        protected long getSystemTableVersion() {
+            if (setOldTimestampToInduceUpgrade) {
+                // Return the next lower version where an upgrade was performed to induce setting the upgradeRequired flag
+                return MetaDataProtocol.getPriorUpgradeVersion();
+            }
+            return MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP;
+        }
+
+        @Override
+        protected PhoenixConnection upgradeSystemCatalogIfRequired(PhoenixConnection metaConnection,
+          long currentServerSideTableTimeStamp) throws InterruptedException, SQLException, TimeoutException, IOException {
+            PhoenixConnection newMetaConnection = super.upgradeSystemCatalogIfRequired(metaConnection, currentServerSideTableTimeStamp);
+            if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP) {
+                actualSysCatUpgrades++;
+            }
+            return newMetaConnection;
+        }
+    }
+
+    public static class PhoenixSysCatCreationTestingDriver extends PhoenixTestDriver {
+        private ConnectionQueryServices cqs;
+        private final ReadOnlyProps overrideProps;
+
+        public PhoenixSysCatCreationTestingDriver(ReadOnlyProps props) {
+            overrideProps = props;
+        }
+
+        @Override // public for testing
+        public synchronized ConnectionQueryServices getConnectionQueryServices(String url, Properties info) throws SQLException {
+            if (cqs == null) {
+                cqs = new PhoenixSysCatCreationServices(new QueryServicesTestImpl(getDefaultProps(), overrideProps), ConnectionInfo.create(url), info);
+                cqs.init(url, info);
+            }
+            return cqs;
+        }
+
+        // NOTE: Do not use this if you want to try re-establishing a connection from the client using a previously
+        // used ConnectionQueryServices instance. This is used only in cases where we need to test server-side
+        // changes and don't care about client-side properties set from the init method.
+        // Reset the Connection Query Services instance so we can create a new connection to the cluster
+        public void resetCQS() {
+            cqs = null;
+        }
+    }
+
+
+    @Before
+    public void resetVariables() {
+        setOldTimestampToInduceUpgrade = false;
+        countUpgradeAttempts = 0;
+        actualSysCatUpgrades = 0;
+    }
+
+    @After
+    public void tearDownMiniCluster() {
+        try {
+            if (testUtil != null) {
+                testUtil.shutdownMiniCluster();
+                testUtil = null;
+            }
+        } catch (Exception e) {
+            // ignore
+        }
+    }
+
+
+     // Conditions: isDoNotUpgradePropSet is true
+     // Expected: We do not create SYSTEM.CATALOG even if this is the first connection to the server
+    @Test
+    public void testFirstConnectionDoNotUpgradePropSet() throws Exception {
+        startMiniClusterWithToggleNamespaceMapping(Boolean.FALSE.toString());
+        Properties propsDoNotUpgradePropSet = new Properties();
+        // Set doNotUpgradeProperty to true
+        UpgradeUtil.doNotUpgradeOnFirstConnection(propsDoNotUpgradePropSet);
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          new SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver(ReadOnlyProps.EMPTY_PROPS);
+        try {
+            driver.getConnectionQueryServices(getJdbcUrl(), propsDoNotUpgradePropSet);
+            fail("Client should not be able to create SYSTEM.CATALOG since we set the doNotUpgrade property");
+        } catch (Exception e) {
+            assertTrue(e instanceof UpgradeRequiredException);
+        }
+        hbaseTables = getHBaseTables();
+        assertFalse(hbaseTables.contains(PHOENIX_SYSTEM_CATALOG) || hbaseTables.contains(PHOENIX_NAMESPACE_MAPPED_SYSTEM_CATALOG));
+        assertTrue(hbaseTables.size() == 0);
+        assertEquals(1, countUpgradeAttempts);
+    }
+
+
+    /********************* Testing SYSTEM.CATALOG/SYSTEM:CATALOG creation/upgrade behavior for subsequent connections *********************/
+
+
+    // Conditions: server-side namespace mapping is enabled, the first connection to the server will create unmapped
+    // SYSTEM tables i.e. SYSTEM\..*, the second connection has client-side namespace mapping enabled and
+    // system table to system namespace mapping enabled
+    // Expected: We will migrate all SYSTEM\..* tables to the SYSTEM namespace
+    @Test
+    public void testMigrateToSystemNamespace() throws Exception {
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          firstConnectionNSMappingServerEnabledClientEnabledMappingDisabled();
+        driver.resetCQS();
+        // Setting this to true to effect migration of SYSTEM tables to the SYSTEM namespace
+        Properties clientProps = getClientProperties(true, true);
+        driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+        hbaseTables = getHBaseTables();
+        assertEquals(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES, hbaseTables);
+        assertEquals(1, countUpgradeAttempts);
+    }
+
+    // Conditions: server-side namespace mapping is enabled, the first connection to the server will create all namespace
+    // mapped SYSTEM tables i.e. SYSTEM:.*, the SYSTEM:CATALOG timestamp at creation is purposefully set to be <
+    // MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP. The subsequent connection has client-side namespace mapping enabled
+    // Expected: An upgrade is attempted when the second client connects to the server
+    @Test
+    public void testUpgradeAttempted() throws Exception {
+        setOldTimestampToInduceUpgrade = true;
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          firstConnectionNSMappingServerEnabledClientEnabled();
+        driver.resetCQS();
+        Properties clientProps = getClientProperties(true, true);
+        setOldTimestampToInduceUpgrade = false;
+        driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+        // There should be no new tables
+        assertEquals(hbaseTables, getHBaseTables());
+        // Since we set an old timestamp on purpose when creating SYSTEM:CATALOG, the second connection attempts to upgrade it
+        assertEquals(1, countUpgradeAttempts);
+        assertEquals(1, actualSysCatUpgrades);
+    }
+
+    // Conditions: server-side namespace mapping is enabled, the first connection to the server will create all namespace
+    // mapped SYSTEM tables i.e. SYSTEM:.*, the SYSTEM:CATALOG timestamp at creation is purposefully set to be <
+    // MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP. The subsequent connection has client-side namespace mapping enabled
+    // Expected: An upgrade is attempted when the second client connects to the server, but this fails since the
+    // isDoNotUpgradePropSet is set to true. We later run EXECUTE UPGRADE manually
+    @Test
+    public void testUpgradeNotAllowed() throws Exception {
+        setOldTimestampToInduceUpgrade = true;
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          firstConnectionNSMappingServerEnabledClientEnabled();
+        driver.resetCQS();
+        Properties clientProps = getClientProperties(true, true);
+        UpgradeUtil.doNotUpgradeOnFirstConnection(clientProps);
+        setOldTimestampToInduceUpgrade = false;
+        try {
+            driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+        } catch (Exception e) {
+            assertTrue(e instanceof UpgradeRequiredException);
+        }
+        // There should be no new tables
+        assertEquals(hbaseTables, getHBaseTables());
+        // Since we set an old timestamp on purpose when creating SYSTEM:CATALOG, the second connection attempts to upgrade it
+        assertEquals(1, countUpgradeAttempts);
+        // This connection is unable to actually upgrade SYSTEM:CATALOG due to isDoNotUpgradePropSet
+        assertEquals(0, actualSysCatUpgrades);
+        Connection conn = driver.getConnectionQueryServices(getJdbcUrl(), new Properties()).connect(getJdbcUrl(), new Properties());
+        try {
+            conn.createStatement().execute(EXECUTE_UPGRADE_COMMAND);
+            // Actually upgraded SYSTEM:CATALOG
+            assertEquals(1, actualSysCatUpgrades);
+        } finally {
+            conn.close();
+        }
+    }
+
+    // Conditions: server-side namespace mapping is enabled, the first connection to the server will create only SYSTEM.CATALOG,
+    // the second connection has client-side namespace mapping enabled
+    // Expected: We will migrate SYSTEM.CATALOG to SYSTEM namespace and create all other SYSTEM:.* tables
+    @Test
+    public void testMigrateSysCatCreateOthers() throws Exception {
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          firstConnectionNSMappingServerEnabledClientDisabled();
+        driver.resetCQS();
+        Properties clientProps = getClientProperties(true, true);
+        driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+        hbaseTables = getHBaseTables();
+        assertEquals(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES, hbaseTables);
+        // SYSTEM.CATALOG migration to the SYSTEM namespace is counted as an upgrade
+        assertEquals(1, countUpgradeAttempts);
+    }
+
+    // Conditions: server-side namespace mapping is enabled, the first connection to the server will create unmapped SYSTEM
+    // tables SYSTEM\..* whose timestamp at creation is purposefully set to be < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP.
+    // The second connection has client-side namespace mapping enabled and system table to system namespace mapping enabled
+    // Expected: We will migrate all SYSTEM\..* tables to the SYSTEM namespace and also upgrade SYSTEM:CATALOG
+    @Test
+    public void testMigrateToSystemNamespaceAndUpgradeSysCat() throws Exception {
+        setOldTimestampToInduceUpgrade = true;
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          firstConnectionNSMappingServerEnabledClientEnabledMappingDisabled();
+        driver.resetCQS();
+        setOldTimestampToInduceUpgrade = false;
+        Properties clientProps = getClientProperties(true, true);
+        driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+        hbaseTables = getHBaseTables();
+        assertEquals(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES, hbaseTables);
+        assertEquals(1, countUpgradeAttempts);
+        assertEquals(1, actualSysCatUpgrades);
+    }
+
+    // Conditions: server-side namespace mapping is enabled, the first connection to the server will create all namespace
+    // mapped SYSTEM tables i.e. SYSTEM:.*, the second connection has client-side namespace mapping disabled
+    // Expected: Throw Inconsistent namespace mapping exception from ensureTableCreated
+    @Test
+    public void testTablesExistInconsistentNSMappingFails() throws Exception {
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          firstConnectionNSMappingServerEnabledClientEnabled();
+        driver.resetCQS();
+        Properties clientProps = getClientProperties(false, false);
+        try {
+            driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+            fail("Client should not be able to connect to cluster with inconsistent client-server namespace mapping properties");
+        } catch (SQLException sqlE) {
+            assertEquals(SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES.getErrorCode(), sqlE.getErrorCode());
+        }
+        hbaseTables = getHBaseTables();
+        assertEquals(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES, hbaseTables);
+        assertEquals(0, countUpgradeAttempts);
+    }
+
+    // Conditions: server-side namespace mapping is enabled, the first connection to the server will create only SYSTEM.CATALOG,
+    // the second connection has client-side namespace mapping disabled
+    // Expected: Throw Inconsistent namespace mapping exception when you check client-server compatibility
+    @Test
+    public void testUnmappedSysCatExistsInconsistentNSMappingFails() throws Exception {
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          firstConnectionNSMappingServerEnabledClientDisabled();
+        driver.resetCQS();
+        Properties clientProps = getClientProperties(false, false);
+        try {
+            driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+            fail("Client should not be able to connect to cluster with inconsistent client-server namespace mapping properties");
+        } catch (SQLException sqlE) {
+            assertEquals(SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES.getErrorCode(), sqlE.getErrorCode());
+        }
+        hbaseTables = getHBaseTables();
+        assertTrue(hbaseTables.contains(PHOENIX_SYSTEM_CATALOG));
+        assertTrue(hbaseTables.size() == 1);
+        assertEquals(0, countUpgradeAttempts);
+    }
+
+    // Conditions: server-side namespace mapping is disabled, the first connection to the server will create all unmapped
+    // SYSTEM tables i.e. SYSTEM\..*, the second connection has client-side namespace mapping enabled
+    // Expected: Throw Inconsistent namespace mapping exception when you check client-server compatibility
+    @Test
+    public void testSysTablesExistInconsistentNSMappingFails() throws Exception {
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          firstConnectionNSMappingServerDisabledClientDisabled();
+        driver.resetCQS();
+        Properties clientProps = getClientProperties(true, true);
+        try {
+            driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+            fail("Client should not be able to connect to cluster with inconsistent client-server namespace mapping properties");
+        } catch (SQLException sqlE) {
+            assertEquals(SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES.getErrorCode(), sqlE.getErrorCode());
+        }
+        hbaseTables = getHBaseTables();
+        assertEquals(PHOENIX_SYSTEM_TABLES, hbaseTables);
+        assertEquals(0, countUpgradeAttempts);
+    }
+
+    // Conditions: server-side namespace mapping is disabled, the first connection to the server will create only SYSTEM:CATALOG
+    // and the second connection has client-side namespace mapping enabled
+    // Expected: Throw Inconsistent namespace mapping exception when you check client-server compatibility
+    @Test
+    public void testMappedSysCatExistsInconsistentNSMappingFails() throws Exception {
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          firstConnectionNSMappingServerDisabledClientEnabled();
+        driver.resetCQS();
+        Properties clientProps = getClientProperties(true, true);
+        try{
+            driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+            fail("Client should not be able to connect to cluster with inconsistent client-server namespace mapping properties");
+        } catch (SQLException sqlE) {
+            assertEquals(SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES.getErrorCode(), sqlE.getErrorCode());
+        }
+        hbaseTables = getHBaseTables();
+        assertTrue(hbaseTables.contains(PHOENIX_NAMESPACE_MAPPED_SYSTEM_CATALOG));
+        assertTrue(hbaseTables.size() == 1);
+        assertEquals(0, countUpgradeAttempts);
+    }
+
+    // Conditions: server-side namespace mapping is disabled, the first connection to the server will create all SYSTEM\..*
+    // tables and the second connection has client-side namespace mapping disabled
+    // Expected: All SYSTEM\..* tables exist and no upgrade is required
+    @Test
+    public void testNSMappingDisabledNoUpgradeRequired() throws Exception {
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          firstConnectionNSMappingServerDisabledClientDisabled();
+        driver.resetCQS();
+        Properties clientProps = getClientProperties(false, false);
+        driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+        hbaseTables = getHBaseTables();
+        assertEquals(PHOENIX_SYSTEM_TABLES, hbaseTables);
+        assertEquals(0, countUpgradeAttempts);
+    }
+
+    // Conditions: server-side namespace mapping is disabled, the first connection to the server will create only SYSTEM:CATALOG
+    // and the second connection has client-side namespace mapping disabled
+    // Expected: The second connection should fail with Inconsistent namespace mapping exception
+    @Test
+    public void testClientNSMappingDisabledConnectionFails() throws Exception {
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          firstConnectionNSMappingServerDisabledClientEnabled();
+        driver.resetCQS();
+        Properties clientProps = getClientProperties(false, false);
+        try{
+            driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+            fail("Client should not be able to connect to cluster with inconsistent client-server namespace mapping properties");
+        } catch (SQLException sqlE) {
+            assertEquals(SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES.getErrorCode(), sqlE.getErrorCode());
+        }
+        hbaseTables = getHBaseTables();
+        assertTrue(hbaseTables.contains(PHOENIX_NAMESPACE_MAPPED_SYSTEM_CATALOG));
+        assertTrue(hbaseTables.size() == 1);
+        assertEquals(0, countUpgradeAttempts);
+    }
+
+    // Conditions: The first connection creates all SYSTEM tables via "EXECUTE UPGRADE" since auto-upgrade is disabled
+    // and the same client alters HBase metadata for SYSTEM.CATALOG
+    // Expected: Another client connection (with a new ConnectionQueryServices instance) made to the server does not
+    // revert the metadata change
+    @Test
+    public void testMetadataAlterRemainsAutoUpgradeDisabled() throws Exception {
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver = firstConnectionAutoUpgradeToggle(false);
+        assertEquals(Integer.parseInt(MODIFIED_MAX_VERSIONS), verifyModificationTableMetadata(driver, PHOENIX_SYSTEM_CATALOG));
+    }
+
+    // Conditions: The first connection creates all SYSTEM tables (auto-upgrade is enabled) and the same client alters
+    // HBase metadata for SYSTEM.CATALOG
+    // Expected: Another client connection (with a new ConnectionQueryServices instance) made to the server does not
+    // revert the metadata change
+    @Test
+    public void testMetadataAlterRemainsAutoUpgradeEnabled() throws Exception {
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver = firstConnectionAutoUpgradeToggle(true);
+        assertEquals(Integer.parseInt(MODIFIED_MAX_VERSIONS), verifyModificationTableMetadata(driver, PHOENIX_SYSTEM_CATALOG));
+    }
+
+    /**
+     * Return all created HBase tables
+     * @return Set of HBase table name strings
+     * @throws IOException
+     */
+    private Set<String> getHBaseTables() throws IOException {
+        Set<String> tables = new HashSet<>();
+        for (TableName tn : testUtil.getHBaseAdmin().listTableNames()) {
+            tables.add(tn.getNameAsString());
+        }
+        return tables;
+    }
+
+    /**
+     * Alter the table metadata and return modified value
+     * @param driver
+     * @param tableName
+     * @return value of VERSIONS option for the table
+     * @throws Exception
+     */
+    private int verifyModificationTableMetadata(PhoenixSysCatCreationTestingDriver driver, String tableName) throws Exception {
+        // Modify table metadata
+        Connection conn = driver.getConnectionQueryServices(getJdbcUrl(), new Properties()).connect(getJdbcUrl(), new Properties());
+        conn.createStatement().execute("ALTER TABLE " + tableName + " SET VERSIONS = " + MODIFIED_MAX_VERSIONS);
+
+        // Connect via a client that creates a new ConnectionQueryServices instance
+        driver.resetCQS();
+        driver.getConnectionQueryServices(getJdbcUrl(), new Properties()).connect(getJdbcUrl(), new Properties());
+        HTableDescriptor descriptor = testUtil.getHBaseAdmin().getTableDescriptor(TableName.valueOf(tableName));
+        return descriptor.getFamily(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES).getMaxVersions();
+    }
+
+    /**
+     * Start the mini-cluster with server-side namespace mapping property specified
+     * @param isNamespaceMappingEnabled
+     * @throws Exception
+     */
+    private void startMiniClusterWithToggleNamespaceMapping(String isNamespaceMappingEnabled) throws Exception {
+        testUtil = new HBaseTestingUtility();
+        Configuration conf = testUtil.getConfiguration();
+        conf.set(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, isNamespaceMappingEnabled);
+        // Avoid multiple clusters trying to bind to the master's info port (16010)
+        conf.setInt(HConstants.MASTER_INFO_PORT, -1);
+        testUtil.startMiniCluster(1);
+    }
+
+    /**
+     * Get the connection string for the mini-cluster
+     * @return Phoenix connection string
+     */
+    private String getJdbcUrl() {
+        return "jdbc:phoenix:localhost:" + testUtil.getZkCluster().getClientPort() + ":/hbase";
+    }
+
+    /**
+     * Set namespace mapping related properties for the client connection
+     * @param nsMappingEnabled
+     * @param systemTableMappingEnabled
+     * @return Properties object
+     */
+    private Properties getClientProperties(boolean nsMappingEnabled, boolean systemTableMappingEnabled) {
+        Properties clientProps = new Properties();
+        clientProps.setProperty(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, Boolean.valueOf(nsMappingEnabled).toString());
+        clientProps.setProperty(QueryServices.IS_SYSTEM_TABLE_MAPPED_TO_NAMESPACE, Boolean.valueOf(systemTableMappingEnabled).toString());
+        return clientProps;
+    }
+
+    /**
+     * Initiate the first connection to the server with provided auto-upgrade property
+     * @param isAutoUpgradeEnabled
+     * @return Phoenix JDBC driver
+     * @throws Exception
+     */
+    private SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver firstConnectionAutoUpgradeToggle(boolean isAutoUpgradeEnabled)
+    throws Exception {
+        if (isAutoUpgradeEnabled) {
+            return firstConnectionNSMappingServerDisabledClientDisabled();
+        }
+        return firstConnectionAutoUpgradeDisabled();
+    }
+
+    // Conditions: isAutoUpgradeEnabled is false
+    // Expected: We do not create SYSTEM.CATALOG even if this is the first connection to the server. Later, when we manually
+    // run "EXECUTE UPGRADE", we create SYSTEM tables
+    private SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver firstConnectionAutoUpgradeDisabled() throws Exception {
+        startMiniClusterWithToggleNamespaceMapping(Boolean.FALSE.toString());
+        Map<String, String> props = new HashMap<>();
+        // Note that the isAutoUpgradeEnabled property is set when instantiating connection query services, not during init
+        props.put(QueryServices.AUTO_UPGRADE_ENABLED, Boolean.FALSE.toString());
+        ReadOnlyProps readOnlyProps = new ReadOnlyProps(props);
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          new SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver(readOnlyProps);
+        try {
+            driver.getConnectionQueryServices(getJdbcUrl(), new Properties());
+            fail("Client should not be able to create SYSTEM.CATALOG since we set the isAutoUpgradeEnabled property to false");
+        } catch (Exception e) {
+            assertTrue(e instanceof UpgradeRequiredException);
+        }
+        hbaseTables = getHBaseTables();
+        assertFalse(hbaseTables.contains(PHOENIX_SYSTEM_CATALOG) || hbaseTables.contains(PHOENIX_NAMESPACE_MAPPED_SYSTEM_CATALOG));
+        assertTrue(hbaseTables.size() == 0);
+        assertEquals(1, countUpgradeAttempts);
+
+        // We use the same ConnectionQueryServices instance to run "EXECUTE UPGRADE"
+        Connection conn = driver.getConnectionQueryServices(getJdbcUrl(), new Properties()).connect(getJdbcUrl(), new Properties());
+        try {
+            conn.createStatement().execute(EXECUTE_UPGRADE_COMMAND);
+        } finally {
+            conn.close();
+        }
+        hbaseTables = getHBaseTables();
+        assertEquals(PHOENIX_SYSTEM_TABLES, hbaseTables);
+        return driver;
+    }
+
+    // Conditions: server-side namespace mapping is enabled, client-side namespace mapping is enabled and system tables
+    // are to be mapped to the SYSTEM namespace.
+    // Expected: If this is the first connection to the server, we should be able to create all namespace mapped system tables i.e. SYSTEM:.*
+    private SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver firstConnectionNSMappingServerEnabledClientEnabled()
+    throws Exception {
+        startMiniClusterWithToggleNamespaceMapping(Boolean.TRUE.toString());
+        Properties clientProps = getClientProperties(true, true);
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          new SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver(ReadOnlyProps.EMPTY_PROPS);
+        driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+        hbaseTables = getHBaseTables();
+        assertEquals(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES, hbaseTables);
+        assertEquals(0, countUpgradeAttempts);
+        return driver;
+    }
+
+    // Conditions: server-side namespace mapping is enabled, client-side namespace mapping is enabled, but mapping
+    // SYSTEM tables to the SYSTEM namespace is disabled
+    // Expected: If this is the first connection to the server, we will create unmapped SYSTEM tables i.e. SYSTEM\..*
+    private SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver firstConnectionNSMappingServerEnabledClientEnabledMappingDisabled()
+    throws Exception {
+        startMiniClusterWithToggleNamespaceMapping(Boolean.TRUE.toString());
+        // client-side namespace mapping is enabled, but mapping SYSTEM tables to SYSTEM namespace is not
+        Properties clientProps = getClientProperties(true, false);
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          new SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver(ReadOnlyProps.EMPTY_PROPS);
+        driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+        hbaseTables = getHBaseTables();
+        assertEquals(PHOENIX_SYSTEM_TABLES, hbaseTables);
+        assertEquals(0, countUpgradeAttempts);
+        return driver;
+    }
+
+    // Conditions: server-side namespace mapping is enabled, client-side namespace mapping is disabled
+    // Expected: Since this is the first connection to the server, we will create SYSTEM.CATALOG but immediately
+    // throw an exception for inconsistent namespace mapping
+    private SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver firstConnectionNSMappingServerEnabledClientDisabled()
+    throws Exception {
+        startMiniClusterWithToggleNamespaceMapping(Boolean.TRUE.toString());
+        Properties clientProps = getClientProperties(false, false);
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          new SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver(ReadOnlyProps.EMPTY_PROPS);
+        try {
+            driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+            fail("Client should not be able to connect to cluster with inconsistent client-server namespace mapping properties");
+        } catch (SQLException sqlE) {
+            assertEquals(SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES.getErrorCode(), sqlE.getErrorCode());
+        }
+        hbaseTables = getHBaseTables();
+        assertTrue(hbaseTables.contains(PHOENIX_SYSTEM_CATALOG));
+        assertTrue(hbaseTables.size() == 1);
+        assertEquals(0, countUpgradeAttempts);
+        return driver;
+    }
+
+    // Conditions: server-side namespace mapping is disabled, client-side namespace mapping is enabled
+    // Expected: Since this is the first connection to the server, we will create the SYSTEM namespace and create
+    // SYSTEM:CATALOG and then immediately throw an exception for inconsistent namespace mapping
+    private SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver firstConnectionNSMappingServerDisabledClientEnabled()
+    throws Exception {
+        startMiniClusterWithToggleNamespaceMapping(Boolean.FALSE.toString());
+        Properties clientProps = getClientProperties(true, true);
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          new SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver(ReadOnlyProps.EMPTY_PROPS);
+        try {
+            driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+            fail("Client should not be able to connect to cluster with inconsistent client-server namespace mapping properties");
+        } catch (SQLException sqlE) {
+            assertEquals(SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES.getErrorCode(), sqlE.getErrorCode());
+        }
+        hbaseTables = getHBaseTables();
+        assertTrue(hbaseTables.contains(PHOENIX_NAMESPACE_MAPPED_SYSTEM_CATALOG));
+        assertTrue(hbaseTables.size() == 1);
+        assertEquals(0, countUpgradeAttempts);
+        return driver;
+    }
+
+    // Conditions: server-side namespace mapping is disabled, client-side namespace mapping is disabled
+    // Expected: Since this is the first connection to the server and auto-upgrade is enabled by default,
+    // we will create all SYSTEM\..* tables
+    private SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver firstConnectionNSMappingServerDisabledClientDisabled()
+    throws Exception {
+        startMiniClusterWithToggleNamespaceMapping(Boolean.FALSE.toString());
+        Properties clientProps = getClientProperties(false, false);
+        SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+          new SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver(ReadOnlyProps.EMPTY_PROPS);
+        driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+        hbaseTables = getHBaseTables();
+        assertEquals(PHOENIX_SYSTEM_TABLES, hbaseTables);
+        assertEquals(0, countUpgradeAttempts);
+        return driver;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index a2d008b..34218d5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -3615,6 +3615,27 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
         }
         long version = MetaDataUtil.encodeVersion(env.getHBaseVersion(), config);
 
+        PTable systemCatalog = null;
+        byte[] tableKey =
+          SchemaUtil.getTableKey(ByteUtil.EMPTY_BYTE_ARRAY,
+            PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA_BYTES,
+            PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE_BYTES);
+        ImmutableBytesPtr cacheKey = new ImmutableBytesPtr(tableKey);
+        try {
+            systemCatalog = loadTable(env, tableKey, cacheKey, MIN_SYSTEM_TABLE_TIMESTAMP,
+              HConstants.LATEST_TIMESTAMP, request.getClientVersion());
+        } catch (Throwable t) {
+            logger.error("loading system catalog table inside getVersion failed", t);
+            ProtobufUtil.setControllerException(controller,
+              ServerUtil.createIOException(
+                SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES,
+                  isTablesMappingEnabled).toString(), t));
+        }
+        // In case this is the first connection, system catalog does not exist, and so we don't
+        // set the optional system catalog timestamp.
+        if (systemCatalog != null) {
+            builder.setSystemCatalogTimestamp(systemCatalog.getTimeStamp());
+        }
         builder.setVersion(version);
         done.run(builder.build());
     }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index a71ce0c..26f8198 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -448,6 +448,10 @@ public abstract class MetaDataProtocol extends MetaDataService {
         return iterator.next();
     }
     
+    public static long getPriorUpgradeVersion() {
+        return TIMESTAMP_VERSION_MAP.lowerKey(TIMESTAMP_VERSION_MAP.lastKey());
+    }
+
     public static String getVersion(long serverTimestamp) {
         /*
          * It is possible that when clients are trying to run upgrades concurrently, we could be at an intermediate

http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
index e041abd..8119c6e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
@@ -14117,6 +14117,16 @@ public final class MetaDataProtos {
      * <code>required int64 version = 1;</code>
      */
     long getVersion();
+
+    // optional int64 systemCatalogTimestamp = 2;
+    /**
+     * <code>optional int64 systemCatalogTimestamp = 2;</code>
+     */
+    boolean hasSystemCatalogTimestamp();
+    /**
+     * <code>optional int64 systemCatalogTimestamp = 2;</code>
+     */
+    long getSystemCatalogTimestamp();
   }
   /**
    * Protobuf type {@code GetVersionResponse}
@@ -14174,6 +14184,11 @@ public final class MetaDataProtos {
               version_ = input.readInt64();
               break;
             }
+            case 16: {
+              bitField0_ |= 0x00000002;
+              systemCatalogTimestamp_ = input.readInt64();
+              break;
+            }
           }
         }
       } catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -14230,8 +14245,25 @@ public final class MetaDataProtos {
       return version_;
     }
 
+    // optional int64 systemCatalogTimestamp = 2;
+    public static final int SYSTEMCATALOGTIMESTAMP_FIELD_NUMBER = 2;
+    private long systemCatalogTimestamp_;
+    /**
+     * <code>optional int64 systemCatalogTimestamp = 2;</code>
+     */
+    public boolean hasSystemCatalogTimestamp() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    /**
+     * <code>optional int64 systemCatalogTimestamp = 2;</code>
+     */
+    public long getSystemCatalogTimestamp() {
+      return systemCatalogTimestamp_;
+    }
+
     private void initFields() {
       version_ = 0L;
+      systemCatalogTimestamp_ = 0L;
     }
     private byte memoizedIsInitialized = -1;
     public final boolean isInitialized() {
@@ -14252,6 +14284,9 @@ public final class MetaDataProtos {
       if (((bitField0_ & 0x00000001) == 0x00000001)) {
         output.writeInt64(1, version_);
       }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeInt64(2, systemCatalogTimestamp_);
+      }
       getUnknownFields().writeTo(output);
     }
 
@@ -14265,6 +14300,10 @@ public final class MetaDataProtos {
         size += com.google.protobuf.CodedOutputStream
           .computeInt64Size(1, version_);
       }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(2, systemCatalogTimestamp_);
+      }
       size += getUnknownFields().getSerializedSize();
       memoizedSerializedSize = size;
       return size;
@@ -14293,6 +14332,11 @@ public final class MetaDataProtos {
         result = result && (getVersion()
             == other.getVersion());
       }
+      result = result && (hasSystemCatalogTimestamp() == other.hasSystemCatalogTimestamp());
+      if (hasSystemCatalogTimestamp()) {
+        result = result && (getSystemCatalogTimestamp()
+            == other.getSystemCatalogTimestamp());
+      }
       result = result &&
           getUnknownFields().equals(other.getUnknownFields());
       return result;
@@ -14310,6 +14354,10 @@ public final class MetaDataProtos {
         hash = (37 * hash) + VERSION_FIELD_NUMBER;
         hash = (53 * hash) + hashLong(getVersion());
       }
+      if (hasSystemCatalogTimestamp()) {
+        hash = (37 * hash) + SYSTEMCATALOGTIMESTAMP_FIELD_NUMBER;
+        hash = (53 * hash) + hashLong(getSystemCatalogTimestamp());
+      }
       hash = (29 * hash) + getUnknownFields().hashCode();
       memoizedHashCode = hash;
       return hash;
@@ -14421,6 +14469,8 @@ public final class MetaDataProtos {
         super.clear();
         version_ = 0L;
         bitField0_ = (bitField0_ & ~0x00000001);
+        systemCatalogTimestamp_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000002);
         return this;
       }
 
@@ -14453,6 +14503,10 @@ public final class MetaDataProtos {
           to_bitField0_ |= 0x00000001;
         }
         result.version_ = version_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.systemCatalogTimestamp_ = systemCatalogTimestamp_;
         result.bitField0_ = to_bitField0_;
         onBuilt();
         return result;
@@ -14472,6 +14526,9 @@ public final class MetaDataProtos {
         if (other.hasVersion()) {
           setVersion(other.getVersion());
         }
+        if (other.hasSystemCatalogTimestamp()) {
+          setSystemCatalogTimestamp(other.getSystemCatalogTimestamp());
+        }
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
@@ -14536,6 +14593,39 @@ public final class MetaDataProtos {
         return this;
       }
 
+      // optional int64 systemCatalogTimestamp = 2;
+      private long systemCatalogTimestamp_ ;
+      /**
+       * <code>optional int64 systemCatalogTimestamp = 2;</code>
+       */
+      public boolean hasSystemCatalogTimestamp() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      /**
+       * <code>optional int64 systemCatalogTimestamp = 2;</code>
+       */
+      public long getSystemCatalogTimestamp() {
+        return systemCatalogTimestamp_;
+      }
+      /**
+       * <code>optional int64 systemCatalogTimestamp = 2;</code>
+       */
+      public Builder setSystemCatalogTimestamp(long value) {
+        bitField0_ |= 0x00000002;
+        systemCatalogTimestamp_ = value;
+        onChanged();
+        return this;
+      }
+      /**
+       * <code>optional int64 systemCatalogTimestamp = 2;</code>
+       */
+      public Builder clearSystemCatalogTimestamp() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        systemCatalogTimestamp_ = 0L;
+        onChanged();
+        return this;
+      }
+
       // @@protoc_insertion_point(builder_scope:GetVersionResponse)
     }
 
@@ -17103,51 +17193,52 @@ public final class MetaDataProtos {
       "cheRequest\022\025\n\rclientVersion\030\001 \001(\005\"*\n\022Cle" +
       "arCacheResponse\022\024\n\014unfreedBytes\030\001 \001(\003\"*\n" +
       "\021GetVersionRequest\022\025\n\rclientVersion\030\001 \001(" +
-      "\005\"%\n\022GetVersionResponse\022\017\n\007version\030\001 \002(\003" +
-      "\"\205\001\n\032ClearTableFromCacheRequest\022\020\n\010tenan",
-      "tId\030\001 \002(\014\022\022\n\nschemaName\030\002 \002(\014\022\021\n\ttableNa" +
-      "me\030\003 \002(\014\022\027\n\017clientTimestamp\030\004 \002(\003\022\025\n\rcli" +
-      "entVersion\030\005 \001(\005\"\035\n\033ClearTableFromCacheR" +
-      "esponse*\365\004\n\014MutationCode\022\030\n\024TABLE_ALREAD" +
-      "Y_EXISTS\020\000\022\023\n\017TABLE_NOT_FOUND\020\001\022\024\n\020COLUM" +
-      "N_NOT_FOUND\020\002\022\031\n\025COLUMN_ALREADY_EXISTS\020\003" +
-      "\022\035\n\031CONCURRENT_TABLE_MUTATION\020\004\022\027\n\023TABLE" +
-      "_NOT_IN_REGION\020\005\022\025\n\021NEWER_TABLE_FOUND\020\006\022" +
-      "\034\n\030UNALLOWED_TABLE_MUTATION\020\007\022\021\n\rNO_PK_C" +
-      "OLUMNS\020\010\022\032\n\026PARENT_TABLE_NOT_FOUND\020\t\022\033\n\027",
-      "FUNCTION_ALREADY_EXISTS\020\n\022\026\n\022FUNCTION_NO" +
-      "T_FOUND\020\013\022\030\n\024NEWER_FUNCTION_FOUND\020\014\022\032\n\026F" +
-      "UNCTION_NOT_IN_REGION\020\r\022\031\n\025SCHEMA_ALREAD" +
-      "Y_EXISTS\020\016\022\026\n\022NEWER_SCHEMA_FOUND\020\017\022\024\n\020SC" +
-      "HEMA_NOT_FOUND\020\020\022\030\n\024SCHEMA_NOT_IN_REGION" +
-      "\020\021\022\032\n\026TABLES_EXIST_ON_SCHEMA\020\022\022\035\n\031UNALLO" +
-      "WED_SCHEMA_MUTATION\020\023\022%\n!AUTO_PARTITION_" +
-      "SEQUENCE_NOT_FOUND\020\024\022#\n\037CANNOT_COERCE_AU" +
-      "TO_PARTITION_ID\020\025\022\024\n\020TOO_MANY_INDEXES\020\0262" +
-      "\345\006\n\017MetaDataService\022/\n\010getTable\022\020.GetTab",
-      "leRequest\032\021.MetaDataResponse\0227\n\014getFunct" +
-      "ions\022\024.GetFunctionsRequest\032\021.MetaDataRes" +
-      "ponse\0221\n\tgetSchema\022\021.GetSchemaRequest\032\021." +
-      "MetaDataResponse\0225\n\013createTable\022\023.Create" +
-      "TableRequest\032\021.MetaDataResponse\022;\n\016creat" +
-      "eFunction\022\026.CreateFunctionRequest\032\021.Meta" +
-      "DataResponse\0227\n\014createSchema\022\024.CreateSch" +
-      "emaRequest\032\021.MetaDataResponse\0221\n\tdropTab" +
-      "le\022\021.DropTableRequest\032\021.MetaDataResponse" +
-      "\0223\n\ndropSchema\022\022.DropSchemaRequest\032\021.Met",
-      "aDataResponse\0227\n\014dropFunction\022\024.DropFunc" +
-      "tionRequest\032\021.MetaDataResponse\0221\n\taddCol" +
-      "umn\022\021.AddColumnRequest\032\021.MetaDataRespons" +
-      "e\0223\n\ndropColumn\022\022.DropColumnRequest\032\021.Me" +
-      "taDataResponse\022?\n\020updateIndexState\022\030.Upd" +
-      "ateIndexStateRequest\032\021.MetaDataResponse\022" +
-      "5\n\nclearCache\022\022.ClearCacheRequest\032\023.Clea" +
-      "rCacheResponse\0225\n\ngetVersion\022\022.GetVersio" +
-      "nRequest\032\023.GetVersionResponse\022P\n\023clearTa" +
-      "bleFromCache\022\033.ClearTableFromCacheReques",
-      "t\032\034.ClearTableFromCacheResponseBB\n(org.a" +
-      "pache.phoenix.coprocessor.generatedB\016Met" +
-      "aDataProtosH\001\210\001\001\240\001\001"
+      "\005\"E\n\022GetVersionResponse\022\017\n\007version\030\001 \002(\003" +
+      "\022\036\n\026systemCatalogTimestamp\030\002 \001(\003\"\205\001\n\032Cle",
+      "arTableFromCacheRequest\022\020\n\010tenantId\030\001 \002(" +
+      "\014\022\022\n\nschemaName\030\002 \002(\014\022\021\n\ttableName\030\003 \002(\014" +
+      "\022\027\n\017clientTimestamp\030\004 \002(\003\022\025\n\rclientVersi" +
+      "on\030\005 \001(\005\"\035\n\033ClearTableFromCacheResponse*" +
+      "\365\004\n\014MutationCode\022\030\n\024TABLE_ALREADY_EXISTS" +
+      "\020\000\022\023\n\017TABLE_NOT_FOUND\020\001\022\024\n\020COLUMN_NOT_FO" +
+      "UND\020\002\022\031\n\025COLUMN_ALREADY_EXISTS\020\003\022\035\n\031CONC" +
+      "URRENT_TABLE_MUTATION\020\004\022\027\n\023TABLE_NOT_IN_" +
+      "REGION\020\005\022\025\n\021NEWER_TABLE_FOUND\020\006\022\034\n\030UNALL" +
+      "OWED_TABLE_MUTATION\020\007\022\021\n\rNO_PK_COLUMNS\020\010",
+      "\022\032\n\026PARENT_TABLE_NOT_FOUND\020\t\022\033\n\027FUNCTION" +
+      "_ALREADY_EXISTS\020\n\022\026\n\022FUNCTION_NOT_FOUND\020" +
+      "\013\022\030\n\024NEWER_FUNCTION_FOUND\020\014\022\032\n\026FUNCTION_" +
+      "NOT_IN_REGION\020\r\022\031\n\025SCHEMA_ALREADY_EXISTS" +
+      "\020\016\022\026\n\022NEWER_SCHEMA_FOUND\020\017\022\024\n\020SCHEMA_NOT" +
+      "_FOUND\020\020\022\030\n\024SCHEMA_NOT_IN_REGION\020\021\022\032\n\026TA" +
+      "BLES_EXIST_ON_SCHEMA\020\022\022\035\n\031UNALLOWED_SCHE" +
+      "MA_MUTATION\020\023\022%\n!AUTO_PARTITION_SEQUENCE" +
+      "_NOT_FOUND\020\024\022#\n\037CANNOT_COERCE_AUTO_PARTI" +
+      "TION_ID\020\025\022\024\n\020TOO_MANY_INDEXES\020\0262\345\006\n\017Meta",
+      "DataService\022/\n\010getTable\022\020.GetTableReques" +
+      "t\032\021.MetaDataResponse\0227\n\014getFunctions\022\024.G" +
+      "etFunctionsRequest\032\021.MetaDataResponse\0221\n" +
+      "\tgetSchema\022\021.GetSchemaRequest\032\021.MetaData" +
+      "Response\0225\n\013createTable\022\023.CreateTableReq" +
+      "uest\032\021.MetaDataResponse\022;\n\016createFunctio" +
+      "n\022\026.CreateFunctionRequest\032\021.MetaDataResp" +
+      "onse\0227\n\014createSchema\022\024.CreateSchemaReque" +
+      "st\032\021.MetaDataResponse\0221\n\tdropTable\022\021.Dro" +
+      "pTableRequest\032\021.MetaDataResponse\0223\n\ndrop",
+      "Schema\022\022.DropSchemaRequest\032\021.MetaDataRes" +
+      "ponse\0227\n\014dropFunction\022\024.DropFunctionRequ" +
+      "est\032\021.MetaDataResponse\0221\n\taddColumn\022\021.Ad" +
+      "dColumnRequest\032\021.MetaDataResponse\0223\n\ndro" +
+      "pColumn\022\022.DropColumnRequest\032\021.MetaDataRe" +
+      "sponse\022?\n\020updateIndexState\022\030.UpdateIndex" +
+      "StateRequest\032\021.MetaDataResponse\0225\n\nclear" +
+      "Cache\022\022.ClearCacheRequest\032\023.ClearCacheRe" +
+      "sponse\0225\n\ngetVersion\022\022.GetVersionRequest" +
+      "\032\023.GetVersionResponse\022P\n\023clearTableFromC",
+      "ache\022\033.ClearTableFromCacheRequest\032\034.Clea" +
+      "rTableFromCacheResponseBB\n(org.apache.ph" +
+      "oenix.coprocessor.generatedB\016MetaDataPro" +
+      "tosH\001\210\001\001\240\001\001"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
       new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -17261,7 +17352,7 @@ public final class MetaDataProtos {
           internal_static_GetVersionResponse_fieldAccessorTable = new
             com.google.protobuf.GeneratedMessage.FieldAccessorTable(
               internal_static_GetVersionResponse_descriptor,
-              new java.lang.String[] { "Version", });
+              new java.lang.String[] { "Version", "SystemCatalogTimestamp", });
           internal_static_ClearTableFromCacheRequest_descriptor =
             getDescriptor().getMessageTypes().get(18);
           internal_static_ClearTableFromCacheRequest_fieldAccessorTable = new

http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/exception/UpgradeRequiredException.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/UpgradeRequiredException.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/UpgradeRequiredException.java
index 9352a50..6c9706b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/UpgradeRequiredException.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/UpgradeRequiredException.java
@@ -17,12 +17,23 @@
  */
 package org.apache.phoenix.exception;
 
+import org.apache.hadoop.hbase.HConstants;
 
 public class UpgradeRequiredException extends RetriableUpgradeException {
+    private final long systemCatalogTimestamp;
 
     public UpgradeRequiredException() {
+        this(HConstants.OLDEST_TIMESTAMP);
+    }
+
+    public UpgradeRequiredException(long systemCatalogTimeStamp) {
         super("Operation not allowed since cluster hasn't been upgraded. Call EXECUTE UPGRADE. ",
-                SQLExceptionCode.UPGRADE_REQUIRED.getSQLState(), SQLExceptionCode.UPGRADE_REQUIRED.getErrorCode());
+          SQLExceptionCode.UPGRADE_REQUIRED.getSQLState(), SQLExceptionCode.UPGRADE_REQUIRED.getErrorCode());
+        this.systemCatalogTimestamp = systemCatalogTimeStamp;
+    }
+
+    public long getSystemCatalogTimeStamp() {
+        return systemCatalogTimestamp;
     }
 
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index 0598a0a..092bfe9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -84,7 +84,7 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
 
     public MetaDataMutationResult createTable(List<Mutation> tableMetaData, byte[] tableName, PTableType tableType,
             Map<String, Object> tableProps, List<Pair<byte[], Map<String, Object>>> families, byte[][] splits,
-            boolean isNamespaceMapped, boolean allocateIndexId) throws SQLException;
+            boolean isNamespaceMapped, boolean allocateIndexId, boolean isDoNotUpgradePropSet) throws SQLException;
     public MetaDataMutationResult dropTable(List<Mutation> tableMetadata, PTableType tableType, boolean cascade) throws SQLException;
     public MetaDataMutationResult dropFunction(List<Mutation> tableMetadata, boolean ifExists) throws SQLException;
     public MetaDataMutationResult addColumn(List<Mutation> tableMetaData, PTable table, Map<String, List<Pair<String,Object>>> properties, Set<String> colFamiliesForPColumnsToBeAdded, List<PColumn> columns) throws SQLException;


[41/50] [abbrv] phoenix git commit: PHOENIX-4686 Phoenix stats does not account for server side limit push downs (Abhishek Chouhan)

Posted by td...@apache.org.
PHOENIX-4686 Phoenix stats does not account for server side limit push downs (Abhishek Chouhan)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/bebe66c4
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/bebe66c4
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/bebe66c4

Branch: refs/heads/system-catalog
Commit: bebe66c4680a7cc3b09a703f8608f966ad4905f1
Parents: 9168f66
Author: James Taylor <jt...@salesforce.com>
Authored: Thu Apr 26 09:14:52 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Thu Apr 26 09:55:28 2018 -0700

----------------------------------------------------------------------
 .../end2end/ExplainPlanWithStatsEnabledIT.java  | 49 ++++++++++++++++++-
 .../org/apache/phoenix/execute/ScanPlan.java    |  4 +-
 .../phoenix/iterate/BaseResultIterators.java    | 50 +++++++++++++++++---
 .../phoenix/iterate/ParallelIterators.java      |  8 ++++
 .../apache/phoenix/iterate/SerialIterators.java | 13 ++---
 .../phoenix/schema/stats/StatisticsUtil.java    |  6 +++
 6 files changed, 111 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/bebe66c4/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
index 2099f4c..abaa2f6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
@@ -40,6 +40,7 @@ import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
 import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTableKey;
 import org.apache.phoenix.schema.TableNotFoundException;
+import org.apache.phoenix.schema.stats.StatisticsUtil;
 import org.apache.phoenix.schema.types.PInteger;
 import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.EnvironmentEdge;
@@ -123,6 +124,50 @@ public class ExplainPlanWithStatsEnabledIT extends ParallelStatsEnabledIT {
     }
 
     @Test
+    public void testBytesRowsForPointSelectWithLimitGreaterThanPointLookupSize() throws Exception {
+        String sql = "SELECT * FROM " + tableA + " where k in (? ,?) limit 4";
+        List<Object> binds = Lists.newArrayList();
+        binds.add(103); binds.add(104);
+        try (Connection conn = DriverManager.getConnection(getUrl())) {
+            Estimate info = getByteRowEstimates(conn, sql, binds);
+            assertEquals((Long) 200L, info.estimatedBytes);
+            assertEquals((Long) 2L, info.estimatedRows);
+            assertEquals((Long) StatisticsUtil.NOT_STATS_BASED_TS, info.estimateInfoTs);
+        }
+    }
+
+    @Test
+    public void testBytesRowsForSelectWithLimit() throws Exception {
+        String sql = "SELECT * FROM " + tableA + " where c1.a in (?,?) limit 3";
+        String noIndexSQL = "SELECT /*+ NO_INDEX */ * FROM " + tableA + " where c1.a in (?,?) limit 3";
+        List<Object> binds = Lists.newArrayList();
+        binds.add(1); binds.add(2);
+        try (Connection conn = DriverManager.getConnection(getUrl())) {
+            Estimate info = getByteRowEstimates(conn, sql, binds);
+            assertEquals((Long) 264L, info.estimatedBytes);
+            assertEquals((Long) 3L, info.estimatedRows);
+            assertEquals((Long) StatisticsUtil.NOT_STATS_BASED_TS, info.estimateInfoTs);
+
+            info = getByteRowEstimates(conn, noIndexSQL, binds);
+            assertEquals((Long) 634L, info.estimatedBytes);
+            assertEquals((Long) 10L, info.estimatedRows);
+            assertTrue(info.estimateInfoTs > 0);
+        }
+    }
+
+    @Test
+    public void testBytesRowsForSelectWithLimitIgnored() throws Exception {
+        String sql = "SELECT * FROM " + tableA + " where (c1.a > c2.b) limit 1";
+        List<Object> binds = Lists.newArrayList();
+        try (Connection conn = DriverManager.getConnection(getUrl())) {
+            Estimate info = getByteRowEstimates(conn, sql, binds);
+            assertEquals((Long) 691L, info.estimatedBytes);
+            assertEquals((Long) 10L, info.estimatedRows);
+            assertTrue(info.estimateInfoTs > 0);
+        }
+    }
+
+    @Test
     public void testBytesRowsForSelectWhenKeyInRange() throws Exception {
         String sql = "SELECT * FROM " + tableB + " where k >= ?";
         List<Object> binds = Lists.newArrayList();
@@ -278,7 +323,7 @@ public class ExplainPlanWithStatsEnabledIT extends ParallelStatsEnabledIT {
             Estimate info = getByteRowEstimates(conn, sql, binds);
             assertEquals((Long) 200L, info.estimatedBytes);
             assertEquals((Long) 2L, info.estimatedRows);
-            assertTrue(info.estimateInfoTs > 0);
+            assertEquals((Long) StatisticsUtil.NOT_STATS_BASED_TS, info.estimateInfoTs);
         }
     }
 
@@ -305,7 +350,7 @@ public class ExplainPlanWithStatsEnabledIT extends ParallelStatsEnabledIT {
             Estimate info = getByteRowEstimates(conn, sql, binds);
             assertEquals((Long) 176L, info.estimatedBytes);
             assertEquals((Long) 2L, info.estimatedRows);
-            assertTrue(info.estimateInfoTs > 0);
+            assertEquals((Long) StatisticsUtil.NOT_STATS_BASED_TS, info.estimateInfoTs);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/bebe66c4/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
index ed145a4..cdb2da5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
@@ -68,8 +68,8 @@ import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTable.IndexType;
 import org.apache.phoenix.schema.SaltingUtil;
 import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.schema.stats.StatisticsUtil;
 import org.apache.phoenix.util.CostUtil;
-import org.apache.phoenix.util.EnvironmentEdgeManager;
 import org.apache.phoenix.util.QueryUtil;
 import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.SchemaUtil;
@@ -124,7 +124,7 @@ public class ScanPlan extends BaseQueryPlan {
         if (isSerial) {
             serialBytesEstimate = estimate.getFirst();
             serialRowsEstimate = estimate.getSecond();
-            serialEstimateInfoTs = EnvironmentEdgeManager.currentTimeMillis();
+            serialEstimateInfoTs = StatisticsUtil.NOT_STATS_BASED_TS;
         }
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/bebe66c4/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 682d1ed..aa9a9f5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
 import org.apache.hadoop.hbase.filter.PageFilter;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -76,6 +77,7 @@ import org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver;
 import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.exception.SQLExceptionInfo;
 import org.apache.phoenix.execute.MutationState;
+import org.apache.phoenix.filter.BooleanExpressionFilter;
 import org.apache.phoenix.filter.ColumnProjectionFilter;
 import org.apache.phoenix.filter.DistinctPrefixFilter;
 import org.apache.phoenix.filter.EncodedQualifiersColumnProjectionFilter;
@@ -170,6 +172,8 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
         return plan.getTableRef().getTable();
     }
     
+    abstract protected boolean isSerial();
+    
     protected boolean useStats() {
         /*
          * Don't use guide posts:
@@ -180,7 +184,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
         if (ScanUtil.isAnalyzeTable(scan)) {
             return false;
         }
-        return true;
+        return !isSerial();
     }
     
     private static void initializeScan(QueryPlan plan, Integer perScanLimit, Integer offset, Scan scan) throws SQLException {
@@ -1105,10 +1109,25 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
                 }
                 regionIndex++;
             }
-            if (scanRanges.isPointLookup()) {
-                this.estimatedRows = Long.valueOf(scanRanges.getPointLookupCount());
+            if (!scans.isEmpty()) { // Add any remaining scans
+                parallelScans.add(scans);
+            }
+            Long pageLimit = getUnfilteredPageLimit(scan);
+            if (scanRanges.isPointLookup() || pageLimit != null) {
+                // If run in parallel, the limit is pushed to each parallel scan so must be accounted for in all of them
+                int parallelFactor = this.isSerial() ? 1 : parallelScans.size();
+                if (scanRanges.isPointLookup() && pageLimit != null) {
+                    this.estimatedRows = Long.valueOf(Math.min(scanRanges.getPointLookupCount(), pageLimit * parallelFactor));
+                } else if (scanRanges.isPointLookup()) {
+                    this.estimatedRows = Long.valueOf(scanRanges.getPointLookupCount());
+                } else {
+                    this.estimatedRows = Long.valueOf(pageLimit) * parallelFactor;
+                }
                 this.estimatedSize = this.estimatedRows * SchemaUtil.estimateRowSize(table);
-                this.estimateInfoTimestamp = computeMinTimestamp(gpsAvailableForAllRegions, estimates, fallbackTs);
+                 // Indication to client that the statistics estimates were not
+                 // calculated based on statistics but instead are based on row
+                 // limits from the query.
+               this.estimateInfoTimestamp = StatisticsUtil.NOT_STATS_BASED_TS;
             } else if (emptyGuidePost) {
                 // In case of an empty guide post, we estimate the number of rows scanned by
                 // using the estimated row size
@@ -1124,9 +1143,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
                 this.estimatedSize = null;
                 this.estimateInfoTimestamp = null;
             }
-            if (!scans.isEmpty()) { // Add any remaining scans
-                parallelScans.add(scans);
-            }
         } finally {
             if (stream != null) Closeables.closeQuietly(stream);
         }
@@ -1134,6 +1150,26 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
         return parallelScans;
     }
 
+    /**
+     * Return row count limit of PageFilter if exists and there is no where
+     * clause filter.
+     * @return
+     */
+    private static Long getUnfilteredPageLimit(Scan scan) {
+        Long pageLimit = null;
+        Iterator<Filter> filters = ScanUtil.getFilterIterator(scan);
+        while (filters.hasNext()) {
+            Filter filter = filters.next();
+            if (filter instanceof BooleanExpressionFilter) {
+                return null;
+            }
+            if (filter instanceof PageFilter) {
+                pageLimit = ((PageFilter)filter).getPageSize();
+            }
+        }
+        return pageLimit;
+    }
+
     private static Long computeMinTimestamp(boolean gpsAvailableForAllRegions, 
             GuidePostEstimate estimates,
             long fallbackTs) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/bebe66c4/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
index 3a4b084..41d278d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
@@ -69,6 +69,14 @@ public class ParallelIterators extends BaseResultIterators {
         this(plan, perScanLimit, iteratorFactory, DefaultParallelScanGrouper.getInstance(), scan, initOneScanPerRegion, caches, dataPlan);
     }  
 
+    /**
+     * No need to use stats when executing serially
+     */
+    @Override
+    protected boolean isSerial() {
+        return false;
+    }
+    
     @Override
     protected void submitWork(final List<List<Scan>> nestedScans, List<List<Pair<Scan,Future<PeekingResultIterator>>>> nestedFutures,
             final Queue<PeekingResultIterator> allIterators, int estFlattenedSize, final boolean isReverse, ParallelScanGrouper scanGrouper) throws SQLException {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/bebe66c4/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
index f94a7c9..c13fcdb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
@@ -73,6 +73,11 @@ public class SerialIterators extends BaseResultIterators {
     }
 
     @Override
+    protected boolean isSerial() {
+        return true;
+    }
+
+    @Override
     protected void submitWork(final List<List<Scan>> nestedScans, List<List<Pair<Scan,Future<PeekingResultIterator>>>> nestedFutures,
             final Queue<PeekingResultIterator> allIterators, int estFlattenedSize, boolean isReverse, final ParallelScanGrouper scanGrouper) {
         ExecutorService executor = context.getConnection().getQueryServices().getExecutor();
@@ -117,14 +122,6 @@ public class SerialIterators extends BaseResultIterators {
         }
     }
 
-    /**
-     * No need to use stats when executing serially
-     */
-    @Override
-    protected boolean useStats() {
-        return false;
-    }
-    
     @Override
     protected String getName() {
         return NAME;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/bebe66c4/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
index 0b9c409..4a758b7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
@@ -47,6 +47,12 @@ import com.google.common.collect.Sets;
  * Simple utility class for managing multiple key parts of the statistic
  */
 public class StatisticsUtil {
+    /**
+     * Indication to client that the statistics estimates were not
+     * calculated based on statistics but instead are based on row
+     * limits from the query.
+     */
+    public static final long NOT_STATS_BASED_TS = 0;
     
     private static final Set<TableName> DISABLE_STATS = Sets.newHashSetWithExpectedSize(8);
     // TODO: make this declarative through new DISABLE_STATS column on SYSTEM.CATALOG table.


[34/50] [abbrv] phoenix git commit: Merge remote-tracking branch 'upstream/master' into PHOENIX-3534

Posted by td...@apache.org.
Merge remote-tracking branch 'upstream/master' into PHOENIX-3534


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d615dc0c
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d615dc0c
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d615dc0c

Branch: refs/heads/system-catalog
Commit: d615dc0cb707d33f58eaeb897269ee074b33c550
Parents: f30994f 78636a3
Author: Thomas D'Silva <td...@apache.org>
Authored: Wed Apr 25 09:11:00 2018 -0700
Committer: Thomas D'Silva <td...@apache.org>
Committed: Wed Apr 25 09:11:00 2018 -0700

----------------------------------------------------------------------
 bin/performance.py                              |    4 +-
 bin/phoenix_utils.py                            |    2 +-
 dev/make_rc.sh                                  |   15 +-
 phoenix-assembly/pom.xml                        |    2 +-
 phoenix-client/pom.xml                          |    2 +-
 phoenix-core/pom.xml                            |   16 +-
 ...ReplayWithIndexWritesAndCompressedWALIT.java |    2 +-
 .../org/apache/phoenix/end2end/AggregateIT.java |  130 +-
 .../apache/phoenix/end2end/AlterTableIT.java    |    2 +-
 .../phoenix/end2end/AlterTableWithViewsIT.java  |    6 +-
 .../phoenix/end2end/AppendOnlySchemaIT.java     |    2 +-
 .../phoenix/end2end/ArrayAppendFunctionIT.java  |   43 +-
 .../phoenix/end2end/ArrayRemoveFunctionIT.java  |  383 ++++
 .../phoenix/end2end/BasePermissionsIT.java      |    4 +
 .../phoenix/end2end/ChangePermissionsIT.java    |   34 +-
 .../phoenix/end2end/CollationKeyFunctionIT.java |   37 +-
 .../end2end/ColumnProjectionOptimizationIT.java |   14 +-
 .../ConnectionQueryServicesTestImpl.java        |   34 +-
 .../phoenix/end2end/CostBasedDecisionIT.java    |  420 +++--
 .../apache/phoenix/end2end/CreateTableIT.java   |    5 +-
 .../org/apache/phoenix/end2end/DeleteIT.java    |  101 ++
 .../org/apache/phoenix/end2end/DropTableIT.java |   43 +
 .../apache/phoenix/end2end/DynamicColumnIT.java |   14 +-
 .../apache/phoenix/end2end/DynamicFamilyIT.java |   26 +-
 .../end2end/ExplainPlanWithStatsEnabledIT.java  |  334 ++--
 .../apache/phoenix/end2end/IndexExtendedIT.java |    6 +-
 .../org/apache/phoenix/end2end/IndexToolIT.java |    6 +-
 .../phoenix/end2end/LocalIndexSplitMergeIT.java |   59 +
 .../org/apache/phoenix/end2end/MapReduceIT.java |   69 +-
 .../phoenix/end2end/MappingTableDataTypeIT.java |    6 +-
 .../MigrateSystemTablesToSystemNamespaceIT.java |   49 +-
 .../phoenix/end2end/MultiCfQueryExecIT.java     |   47 +
 .../end2end/NamespaceSchemaMappingIT.java       |   19 +-
 .../phoenix/end2end/NativeHBaseTypesIT.java     |   30 +-
 .../org/apache/phoenix/end2end/OrderByIT.java   |  129 +-
 .../phoenix/end2end/ParallelStatsEnabledIT.java |    1 +
 .../PartialScannerResultsDisabledIT.java        |    2 +-
 .../apache/phoenix/end2end/PhoenixDriverIT.java |   99 +-
 .../end2end/QueryDatabaseMetaDataIT.java        |   39 +-
 .../apache/phoenix/end2end/QueryLoggerIT.java   |  358 ++++
 .../apache/phoenix/end2end/ReverseScanIT.java   |   88 +-
 .../phoenix/end2end/SortMergeJoinMoreIT.java    |  116 +-
 .../phoenix/end2end/SpillableGroupByIT.java     |   74 +-
 .../SystemCatalogCreationOnConnectionIT.java    |  626 +++++++
 .../end2end/SystemTablePermissionsIT.java       |    2 +-
 .../end2end/TenantSpecificTablesDDLIT.java      |    2 +
 .../UngroupedAggregateRegionObserverIT.java     |  171 --
 .../org/apache/phoenix/end2end/UpgradeIT.java   |    2 +-
 .../end2end/UpsertSelectAutoCommitIT.java       |   28 +
 .../phoenix/end2end/UserDefinedFunctionsIT.java |  165 +-
 .../phoenix/end2end/index/BaseIndexIT.java      |   22 +
 .../index/ChildViewsUseParentViewIndexIT.java   |   10 +-
 .../phoenix/end2end/index/ImmutableIndexIT.java |    3 +
 .../end2end/index/MutableIndexFailureIT.java    |   27 +-
 .../phoenix/end2end/index/MutableIndexIT.java   |   95 +
 .../end2end/index/MutableIndexRebuilderIT.java  |  143 ++
 .../end2end/index/PartialIndexRebuilderIT.java  |  130 +-
 .../phoenix/end2end/index/ViewIndexIT.java      |   45 +-
 .../end2end/index/txn/TxWriteFailureIT.java     |    6 +-
 .../apache/phoenix/end2end/join/BaseJoinIT.java |    2 +
 .../phoenix/end2end/join/HashJoinMoreIT.java    |    4 +-
 .../apache/phoenix/end2end/join/SubqueryIT.java |   49 +-
 .../join/SubqueryUsingSortMergeJoinIT.java      |   12 +-
 .../phoenix/monitoring/PhoenixMetricsIT.java    |   24 +-
 .../phoenix/tx/FlappingTransactionIT.java       |   11 +-
 .../phoenix/tx/ParameterizedTransactionIT.java  |   18 +-
 .../org/apache/phoenix/tx/TransactionIT.java    |   12 +
 .../org/apache/phoenix/tx/TxCheckpointIT.java   |    5 +-
 .../hadoop/hbase/ipc/PhoenixRpcScheduler.java   |   30 +
 .../controller/ClientRpcControllerFactory.java  |   18 +-
 .../ipc/controller/IndexRpcController.java      |    8 +-
 ...erRegionServerIndexRpcControllerFactory.java |   18 +-
 ...egionServerMetadataRpcControllerFactory.java |   17 +-
 .../ipc/controller/MetadataRpcController.java   |    8 +-
 .../IndexHalfStoreFileReaderGenerator.java      |   30 +-
 .../LocalIndexStoreFileScanner.java             |    4 +-
 .../hadoop/hbase/regionserver/ScanInfoUtil.java |   12 +
 .../org/apache/phoenix/cache/GlobalCache.java   |    6 +-
 .../org/apache/phoenix/cache/HashCache.java     |    1 +
 .../phoenix/cache/IndexMetaDataCache.java       |    7 +
 .../apache/phoenix/cache/ServerCacheClient.java |    7 +-
 .../org/apache/phoenix/cache/TenantCache.java   |    2 +-
 .../apache/phoenix/cache/TenantCacheImpl.java   |    4 +-
 .../apache/phoenix/compile/DeleteCompiler.java  |  124 +-
 .../org/apache/phoenix/compile/ExplainPlan.java |   10 +
 .../apache/phoenix/compile/FromCompiler.java    |    2 +-
 .../apache/phoenix/compile/GroupByCompiler.java |   19 +-
 .../apache/phoenix/compile/JoinCompiler.java    |  324 ++--
 .../phoenix/compile/ListJarsQueryPlan.java      |    8 +-
 .../apache/phoenix/compile/OrderByCompiler.java |   11 +-
 .../phoenix/compile/OrderPreservingTracker.java |   13 +
 .../apache/phoenix/compile/PostDDLCompiler.java |    2 +-
 .../apache/phoenix/compile/QueryCompiler.java   |  498 +++---
 .../org/apache/phoenix/compile/QueryPlan.java   |    2 +
 .../org/apache/phoenix/compile/ScanRanges.java  |   17 +-
 .../apache/phoenix/compile/SequenceManager.java |    2 +-
 .../phoenix/compile/StatementContext.java       |   10 +
 .../phoenix/compile/SubselectRewriter.java      |    5 +
 .../apache/phoenix/compile/TraceQueryPlan.java  |    6 +
 .../compile/TupleProjectionCompiler.java        |   44 +-
 .../apache/phoenix/compile/UnionCompiler.java   |    6 +-
 .../apache/phoenix/compile/UpsertCompiler.java  |   47 +-
 .../apache/phoenix/compile/WhereOptimizer.java  |   23 +-
 .../coprocessor/BaseScannerRegionObserver.java  |   15 +-
 .../DelegateRegionCoprocessorEnvironment.java   |   15 +-
 .../coprocessor/DelegateRegionObserver.java     |   32 +-
 .../GroupedAggregateRegionObserver.java         |    4 +
 .../coprocessor/MetaDataEndpointImpl.java       |  160 +-
 .../phoenix/coprocessor/MetaDataProtocol.java   |   16 +-
 .../coprocessor/MetaDataRegionObserver.java     |   21 +-
 .../coprocessor/PhoenixAccessController.java    |  102 +-
 .../PhoenixMetaDataCoprocessorHost.java         |    6 +
 .../PhoenixTransactionalProcessor.java          |   28 -
 .../phoenix/coprocessor/ScanRegionObserver.java |    4 +-
 .../coprocessor/ServerCachingEndpointImpl.java  |    4 +-
 .../coprocessor/ServerCachingProtocol.java      |    2 +-
 .../TephraTransactionalProcessor.java           |   29 +
 .../UngroupedAggregateRegionObserver.java       |  311 ++--
 .../coprocessor/generated/MetaDataProtos.java   |  183 +-
 .../coprocessor/generated/PTableProtos.java     |  110 +-
 .../generated/ServerCachingProtos.java          |  117 +-
 .../phoenix/exception/SQLExceptionCode.java     |   17 +-
 .../exception/UpgradeRequiredException.java     |   13 +-
 .../apache/phoenix/execute/AggregatePlan.java   |   53 +-
 .../apache/phoenix/execute/BaseQueryPlan.java   |   21 +-
 .../phoenix/execute/ClientAggregatePlan.java    |   46 +-
 .../phoenix/execute/ClientProcessingPlan.java   |    4 +
 .../apache/phoenix/execute/ClientScanPlan.java  |   22 +-
 .../apache/phoenix/execute/CorrelatePlan.java   |   26 +-
 .../apache/phoenix/execute/CursorFetchPlan.java |    6 +
 .../apache/phoenix/execute/DelegateHTable.java  |   20 +
 .../apache/phoenix/execute/HashJoinPlan.java    |  128 +-
 .../execute/LiteralResultIterationPlan.java     |    8 +-
 .../apache/phoenix/execute/MutationState.java   |  219 ++-
 .../PhoenixTxIndexMutationGenerator.java        |  443 +++++
 .../org/apache/phoenix/execute/ScanPlan.java    |   32 +-
 .../phoenix/execute/SortMergeJoinPlan.java      |   20 +-
 .../phoenix/execute/TupleProjectionPlan.java    |    6 +
 .../org/apache/phoenix/execute/UnionPlan.java   |   12 +-
 .../apache/phoenix/execute/UnnestArrayPlan.java |    6 +
 .../execute/visitor/AvgRowWidthVisitor.java     |  205 +++
 .../execute/visitor/ByteCountVisitor.java       |  125 ++
 .../execute/visitor/QueryPlanVisitor.java       |   46 +
 .../execute/visitor/RowCountVisitor.java        |  335 ++++
 .../expression/BaseSingleExpression.java        |    5 +
 .../phoenix/expression/ExpressionType.java      |  119 +-
 .../expression/aggregator/Aggregators.java      |    3 +-
 .../aggregator/ClientAggregators.java           |    3 +-
 .../DistinctValueWithCountServerAggregator.java |   20 +-
 .../aggregator/ServerAggregators.java           |    6 +-
 .../function/ArrayAppendFunction.java           |   11 +-
 .../function/ArrayConcatFunction.java           |    3 +-
 .../function/ArrayModifierFunction.java         |   55 +-
 .../function/ArrayPrependFunction.java          |    9 +-
 .../function/ArrayRemoveFunction.java           |   88 +
 .../function/CollationKeyFunction.java          |   24 +-
 .../expression/function/LowerFunction.java      |   28 +-
 .../expression/function/ScalarFunction.java     |   15 +-
 .../TransactionProviderNameFunction.java        |   81 +
 .../expression/function/UDFExpression.java      |   39 +-
 .../expression/function/UpperFunction.java      |   29 +-
 .../org/apache/phoenix/hbase/index/Indexer.java |   25 +-
 .../hbase/index/builder/BaseIndexBuilder.java   |    4 +-
 .../hbase/index/builder/BaseIndexCodec.java     |    7 -
 .../hbase/index/builder/IndexBuildManager.java  |    4 +
 .../phoenix/hbase/index/covered/IndexCodec.java |   14 +-
 .../hbase/index/covered/IndexMetaData.java      |   12 +-
 .../hbase/index/covered/LocalTableState.java    |   16 +-
 .../hbase/index/covered/NonTxIndexBuilder.java  |    2 +-
 .../phoenix/hbase/index/covered/TableState.java |   15 -
 .../index/exception/IndexWriteException.java    |   52 +-
 .../MultiIndexWriteFailureException.java        |   39 +-
 .../SingleIndexWriteFailureException.java       |   32 +-
 .../hbase/index/scanner/ScannerBuilder.java     |   16 +
 .../hbase/index/write/IndexCommitter.java       |    2 +-
 .../phoenix/hbase/index/write/IndexWriter.java  |   21 +-
 .../hbase/index/write/IndexWriterUtils.java     |   87 +-
 .../write/ParallelWriterIndexCommitter.java     |   19 +-
 .../hbase/index/write/RecoveryIndexWriter.java  |    4 +-
 .../TrackingParallelWriterIndexCommitter.java   |   20 +-
 .../apache/phoenix/index/IndexMaintainer.java   |   39 +-
 .../index/IndexMetaDataCacheFactory.java        |    9 +-
 .../phoenix/index/PhoenixIndexBuilder.java      |   21 +-
 .../apache/phoenix/index/PhoenixIndexCodec.java |   36 +-
 .../index/PhoenixIndexFailurePolicy.java        |  207 ++-
 .../phoenix/index/PhoenixIndexMetaData.java     |   74 +-
 .../index/PhoenixIndexMetaDataBuilder.java      |  107 ++
 .../index/PhoenixTransactionalIndexer.java      |  469 +----
 .../phoenix/iterate/BaseResultIterators.java    |  432 ++++-
 .../apache/phoenix/iterate/ExplainTable.java    |    3 +-
 .../NonAggregateRegionScannerFactory.java       |   61 +-
 .../phoenix/iterate/ParallelIterators.java      |    8 +-
 .../phoenix/iterate/RegionScannerFactory.java   |    4 +-
 .../phoenix/iterate/ScanningResultIterator.java |   26 +-
 .../apache/phoenix/iterate/SerialIterators.java |    4 +-
 .../apache/phoenix/iterate/SnapshotScanner.java |   12 +-
 .../phoenix/iterate/TableResultIterator.java    |    7 +-
 .../apache/phoenix/jdbc/PhoenixConnection.java  |   19 +-
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |   35 +-
 .../phoenix/jdbc/PhoenixPreparedStatement.java  |   11 +-
 .../apache/phoenix/jdbc/PhoenixResultSet.java   |   38 +
 .../apache/phoenix/jdbc/PhoenixStatement.java   |   85 +-
 .../apache/phoenix/join/HashCacheFactory.java   |   34 +-
 .../java/org/apache/phoenix/log/LogLevel.java   |   22 +
 .../java/org/apache/phoenix/log/LogWriter.java  |   51 +
 .../log/QueryLogDetailsEventHandler.java        |   63 +
 .../org/apache/phoenix/log/QueryLogInfo.java    |   87 +
 .../org/apache/phoenix/log/QueryLogState.java   |   22 +
 .../org/apache/phoenix/log/QueryLogger.java     |  145 ++
 .../log/QueryLoggerDefaultExceptionHandler.java |   51 +
 .../phoenix/log/QueryLoggerDisruptor.java       |  117 ++
 .../org/apache/phoenix/log/QueryLoggerUtil.java |   62 +
 .../org/apache/phoenix/log/RingBufferEvent.java |   93 +
 .../phoenix/log/RingBufferEventTranslator.java  |   53 +
 .../org/apache/phoenix/log/TableLogWriter.java  |  125 ++
 .../phoenix/mapreduce/PhoenixInputFormat.java   |   51 +-
 .../index/PhoenixIndexPartialBuildMapper.java   |    5 +
 .../util/PhoenixConfigurationUtil.java          |   63 +-
 .../mapreduce/util/PhoenixMapReduceUtil.java    |    5 +-
 .../phoenix/memory/ChildMemoryManager.java      |    7 +-
 .../phoenix/memory/GlobalMemoryManager.java     |   19 +-
 .../phoenix/monitoring/GlobalClientMetrics.java |   27 +-
 .../phoenix/monitoring/ReadMetricQueue.java     |   44 +-
 .../phoenix/monitoring/ScanMetricsHolder.java   |   48 +-
 .../GenSubqueryParamValuesRewriter.java         |  153 ++
 .../apache/phoenix/optimize/QueryOptimizer.java |  212 ++-
 .../phoenix/parse/ArrayModifierParseNode.java   |   36 +
 .../apache/phoenix/parse/FunctionParseNode.java |   12 +-
 .../java/org/apache/phoenix/parse/HintNode.java |    4 +
 .../org/apache/phoenix/parse/ParseNode.java     |   26 +-
 .../phoenix/query/ConnectionQueryServices.java  |   12 +-
 .../query/ConnectionQueryServicesImpl.java      |  985 ++++++-----
 .../query/ConnectionlessQueryServicesImpl.java  |   77 +-
 .../query/DelegateConnectionQueryServices.java  |   23 +-
 .../apache/phoenix/query/QueryConstants.java    |  136 +-
 .../org/apache/phoenix/query/QueryServices.java |   16 +-
 .../phoenix/query/QueryServicesOptions.java     |   29 +-
 .../apache/phoenix/schema/DelegateTable.java    |    8 +-
 .../apache/phoenix/schema/MetaDataClient.java   |  335 ++--
 .../org/apache/phoenix/schema/PColumnImpl.java  |    6 +-
 .../org/apache/phoenix/schema/PIndexState.java  |    7 +-
 .../apache/phoenix/schema/PMetaDataImpl.java    |    2 +-
 .../java/org/apache/phoenix/schema/PTable.java  |    2 +
 .../org/apache/phoenix/schema/PTableImpl.java   |   92 +-
 .../phoenix/schema/RowKeyValueAccessor.java     |    2 +-
 .../apache/phoenix/schema/TableProperty.java    |   18 +
 .../schema/TablesNotInSyncException.java        |   17 +
 .../phoenix/schema/stats/StatisticsWriter.java  |    9 +-
 .../transaction/OmidTransactionContext.java     |   57 +-
 .../transaction/OmidTransactionProvider.java    |   84 +
 .../transaction/OmidTransactionTable.java       |  364 ----
 .../transaction/PhoenixTransactionClient.java   |   23 +
 .../transaction/PhoenixTransactionContext.java  |  169 +-
 .../transaction/PhoenixTransactionProvider.java |   51 +
 .../transaction/PhoenixTransactionService.java  |   24 +
 .../transaction/PhoenixTransactionalTable.java  |  149 --
 .../transaction/TephraTransactionContext.java   |  205 +--
 .../transaction/TephraTransactionProvider.java  |  197 +++
 .../transaction/TephraTransactionTable.java     |  350 ----
 .../phoenix/transaction/TransactionFactory.java |  139 +-
 .../java/org/apache/phoenix/util/CostUtil.java  |   61 +-
 .../org/apache/phoenix/util/ExpressionUtil.java |    2 +-
 .../java/org/apache/phoenix/util/IndexUtil.java |    6 +-
 .../org/apache/phoenix/util/KeyValueUtil.java   |   12 +
 .../org/apache/phoenix/util/PhoenixRuntime.java |   27 +-
 .../java/org/apache/phoenix/util/QueryUtil.java |   14 +-
 .../java/org/apache/phoenix/util/ScanUtil.java  |   29 +-
 .../org/apache/phoenix/util/SchemaUtil.java     |    6 +
 .../org/apache/phoenix/util/ServerUtil.java     |   95 +-
 .../java/org/apache/phoenix/util/SizedUtil.java |    2 +-
 .../apache/phoenix/util/TransactionUtil.java    |   93 +-
 .../org/apache/phoenix/util/UpgradeUtil.java    |   40 +-
 .../wal/ReadWriteKeyValuesWithCodecTest.java    |    6 +-
 .../apache/phoenix/cache/TenantCacheTest.java   |   14 +-
 .../phoenix/compile/QueryCompilerTest.java      |  815 ++++++++-
 .../phoenix/compile/QueryOptimizerTest.java     |   41 +
 .../phoenix/compile/WhereOptimizerTest.java     |  235 +++
 .../phoenix/execute/CorrelatePlanTest.java      |    4 +-
 .../execute/LiteralResultIteratorPlanTest.java  |    4 +-
 .../expression/ArrayRemoveFunctionTest.java     |  285 +++
 .../function/CollationKeyFunctionTest.java      |   14 +-
 .../expression/function/LowerFunctionTest.java  |   93 +
 .../expression/function/UpperFunctionTest.java  |   92 +
 .../index/covered/CoveredColumnIndexCodec.java  |    8 +-
 .../covered/CoveredIndexCodecForTesting.java    |    5 +-
 .../index/covered/LocalTableStateTest.java      |   38 +-
 .../index/covered/NonTxIndexBuilderTest.java    |    3 +
 .../covered/TestCoveredColumnIndexCodec.java    |   12 +-
 .../hbase/index/write/TestIndexWriter.java      |   25 +-
 .../index/write/TestParalleIndexWriter.java     |   12 +-
 .../write/TestParalleWriterIndexCommitter.java  |   11 +-
 .../index/write/TestWALRecoveryCaching.java     |    7 +-
 .../recovery/TestPerRegionIndexWriteCache.java  |    4 +-
 .../phoenix/memory/MemoryManagerTest.java       |   15 +
 .../java/org/apache/phoenix/query/BaseTest.java |   26 +-
 .../query/ConnectionQueryServicesImplTest.java  |   11 +-
 .../apache/phoenix/query/KeyRangeClipTest.java  |  155 ++
 .../query/ParallelIteratorsSplitTest.java       |    8 +-
 .../phoenix/query/QueryServicesTestImpl.java    |   15 +-
 .../phoenix/schema/PMetaDataImplTest.java       |   21 +-
 .../java/org/apache/phoenix/util/TestUtil.java  |   54 +-
 phoenix-flume/pom.xml                           |    8 +-
 phoenix-hive/pom.xml                            |    2 +-
 .../hive/mapreduce/PhoenixInputFormat.java      |    3 +-
 phoenix-kafka/pom.xml                           |   12 +-
 phoenix-load-balancer/pom.xml                   |    2 +-
 phoenix-pherf/pom.xml                           |    6 +-
 .../phoenix/pherf/configuration/Column.java     |   23 +-
 .../pherf/configuration/DataTypeMapping.java    |    6 +-
 .../phoenix/pherf/configuration/Query.java      |   25 +
 .../phoenix/pherf/configuration/Scenario.java   |    3 +-
 .../phoenix/pherf/result/QueryResult.java       |   10 +-
 .../phoenix/pherf/result/ResultManager.java     |   18 +-
 .../apache/phoenix/pherf/result/ResultUtil.java |    9 +-
 .../phoenix/pherf/rules/RulesApplier.java       |  142 +-
 .../phoenix/pherf/schema/SchemaReader.java      |    2 +-
 .../apache/phoenix/pherf/util/PhoenixUtil.java  |   44 +-
 .../pherf/workload/MultiThreadedRunner.java     |   35 +-
 .../phoenix/pherf/workload/QueryExecutor.java   |   36 +-
 .../phoenix/pherf/workload/WriteWorkload.java   |   54 +-
 .../scenario/prod_test_unsalted_scenario.xml    |   14 +-
 .../org/apache/phoenix/pherf/ColumnTest.java    |    3 +
 .../phoenix/pherf/ConfigurationParserTest.java  |    4 +-
 .../apache/phoenix/pherf/RuleGeneratorTest.java |  107 +-
 .../test/resources/datamodel/test_schema.sql    |    4 +
 .../test/resources/scenario/test_scenario.xml   |   48 +-
 phoenix-pig/pom.xml                             |    6 +-
 phoenix-protocol/src/main/MetaDataService.proto |    1 +
 phoenix-protocol/src/main/PTable.proto          |    1 +
 .../src/main/ServerCachingService.proto         |    1 +
 phoenix-queryserver-client/pom.xml              |    6 +-
 phoenix-queryserver/pom.xml                     |    6 +-
 .../HttpParamImpersonationQueryServerIT.java    |   16 +-
 .../phoenix/end2end/SecureQueryServerIT.java    |   15 +-
 .../phoenix/queryserver/server/QueryServer.java |   21 +-
 phoenix-server/pom.xml                          |    2 +-
 phoenix-spark/pom.xml                           |    2 +-
 .../phoenix/spark/DataFrameFunctions.scala      |    6 +-
 .../phoenix/spark/ProductRDDFunctions.scala     |    6 +-
 phoenix-tracing-webapp/pom.xml                  |    2 +-
 pom.xml                                         |   35 +-
 python/.gitignore                               |    8 +
 python/.gitlab-ci.yml                           |  149 ++
 python/NEWS.rst                                 |   44 +
 python/README.rst                               |  136 ++
 python/RELEASING.rst                            |   12 +
 python/ci/build-env/Dockerfile                  |    7 +
 python/ci/phoenix/Dockerfile                    |   33 +
 python/ci/phoenix/docker-entrypoint.sh          |   24 +
 python/ci/phoenix/hbase-site.xml                |   12 +
 python/doc/Makefile                             |  192 ++
 python/doc/api.rst                              |   30 +
 python/doc/conf.py                              |  287 +++
 python/doc/index.rst                            |   27 +
 python/doc/versions.rst                         |    3 +
 python/docker-compose.yml                       |   21 +
 python/examples/basic.py                        |   27 +
 python/examples/shell.py                        |   33 +
 python/gen-protobuf.sh                          |   38 +
 python/phoenixdb/__init__.py                    |   68 +
 python/phoenixdb/avatica/__init__.py            |   16 +
 python/phoenixdb/avatica/client.py              |  510 ++++++
 python/phoenixdb/avatica/proto/__init__.py      |    0
 python/phoenixdb/avatica/proto/common_pb2.py    | 1667 ++++++++++++++++++
 python/phoenixdb/avatica/proto/requests_pb2.py  | 1206 +++++++++++++
 python/phoenixdb/avatica/proto/responses_pb2.py |  917 ++++++++++
 python/phoenixdb/connection.py                  |  187 ++
 python/phoenixdb/cursor.py                      |  347 ++++
 python/phoenixdb/errors.py                      |   93 +
 python/phoenixdb/tests/__init__.py              |   44 +
 python/phoenixdb/tests/dbapi20.py               |  857 +++++++++
 python/phoenixdb/tests/test_avatica.py          |   25 +
 python/phoenixdb/tests/test_connection.py       |   42 +
 python/phoenixdb/tests/test_db.py               |   99 ++
 python/phoenixdb/tests/test_dbapi20.py          |  122 ++
 python/phoenixdb/tests/test_errors.py           |   60 +
 python/phoenixdb/tests/test_types.py            |  327 ++++
 python/phoenixdb/types.py                       |  202 +++
 python/requirements.txt                         |   20 +
 python/setup.cfg                                |   34 +
 python/setup.py                                 |   64 +
 python/tox.ini                                  |   24 +
 382 files changed, 22540 insertions(+), 5304 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
index cbc1021,627e453..7960543
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
@@@ -59,10 -63,10 +63,10 @@@ public class MigrateSystemTablesToSyste
  
      private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
              "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
-             "SYSTEM.MUTEX", "SYSTEM.CHILD_LINK"));
+             "SYSTEM.MUTEX","SYSTEM.LOG"));
      private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
              Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
-                     "SYSTEM:MUTEX", "SYSTEM:CHILD_LINK"));
 -                    "SYSTEM:MUTEX","SYSTEM:LOG"));
++                    "SYSTEM:MUTEX","SYSTEM:LOG", "SYSTEM:CHILD_LINK"));
      private static final String SCHEMA_NAME = "MIGRATETEST";
      private static final String TABLE_NAME =
              SCHEMA_NAME + "." + MigrateSystemTablesToSystemNamespaceIT.class.getSimpleName().toUpperCase();

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
index 6830d54,34a1312..41beaa1
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
@@@ -491,10 -491,10 +491,12 @@@ public class TenantSpecificTablesDDLIT 
              assertTrue(rs.next());
              assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE, PTableType.SYSTEM);
              assertTrue(rs.next());
 +            assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_TABLE, PTableType.SYSTEM);
 +            assertTrue(rs.next());
              assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, SYSTEM_FUNCTION_TABLE, SYSTEM);
              assertTrue(rs.next());
+             assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE, PTableType.SYSTEM);
+             assertTrue(rs.next());
              assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.TYPE_SEQUENCE, PTableType.SYSTEM);
              assertTrue(rs.next());
              assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE, PTableType.SYSTEM);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexIT.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ChildViewsUseParentViewIndexIT.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 6277b78,4c72c2d..06c9aa9
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@@ -93,11 -95,8 +94,10 @@@ import java.util.Collections
  import java.util.Comparator;
  import java.util.HashSet;
  import java.util.Iterator;
- import java.util.LinkedList;
  import java.util.List;
 +import java.util.ListIterator;
  import java.util.Map;
 +import java.util.Map.Entry;
  import java.util.NavigableMap;
  import java.util.Set;
  
@@@ -213,7 -216,9 +212,6 @@@ import org.apache.phoenix.schema.Sequen
  import org.apache.phoenix.schema.SequenceNotFoundException;
  import org.apache.phoenix.schema.SortOrder;
  import org.apache.phoenix.schema.TableNotFoundException;
--import org.apache.phoenix.schema.TableProperty;
 -import org.apache.phoenix.schema.TableRef;
 -import org.apache.phoenix.schema.tuple.ResultTuple;
  import org.apache.phoenix.schema.types.PBinary;
  import org.apache.phoenix.schema.types.PBoolean;
  import org.apache.phoenix.schema.types.PDataType;
@@@ -550,30 -557,30 +552,35 @@@ public class MetaDataEndpointImpl exten
                      TableName.valueOf(table.getPhysicalName().getBytes()));
  
              builder.setReturnCode(MetaDataProtos.MutationCode.TABLE_ALREADY_EXISTS);
-             long disableIndexTimestamp = table.getIndexDisableTimestamp();
-             long minNonZerodisableIndexTimestamp = disableIndexTimestamp > 0 ? disableIndexTimestamp : Long.MAX_VALUE;
-             for (PTable index : table.getIndexes()) {
-                 disableIndexTimestamp = index.getIndexDisableTimestamp();
-                 if (disableIndexTimestamp > 0 && (index.getIndexState() == PIndexState.ACTIVE || index.getIndexState() == PIndexState.PENDING_ACTIVE) && disableIndexTimestamp < minNonZerodisableIndexTimestamp) {
-                     minNonZerodisableIndexTimestamp = disableIndexTimestamp;
+             builder.setMutationTime(currentTime);
+             if (blockWriteRebuildIndex) {
+                 long disableIndexTimestamp = table.getIndexDisableTimestamp();
+                 long minNonZerodisableIndexTimestamp = disableIndexTimestamp > 0 ? disableIndexTimestamp : Long.MAX_VALUE;
+                 for (PTable index : table.getIndexes()) {
+                     disableIndexTimestamp = index.getIndexDisableTimestamp();
+                     if (disableIndexTimestamp > 0
+                             && (index.getIndexState() == PIndexState.ACTIVE
+                                     || index.getIndexState() == PIndexState.PENDING_ACTIVE
+                                     || index.getIndexState() == PIndexState.PENDING_DISABLE)
+                             && disableIndexTimestamp < minNonZerodisableIndexTimestamp) {
+                         minNonZerodisableIndexTimestamp = disableIndexTimestamp;
+                     }
+                 }
+                 // Freeze time for table at min non-zero value of INDEX_DISABLE_TIMESTAMP
+                 // This will keep the table consistent with index as the table has had one more
+                 // batch applied to it.
+                 if (minNonZerodisableIndexTimestamp != Long.MAX_VALUE) {
+                     // Subtract one because we add one due to timestamp granularity in Windows
+                     builder.setMutationTime(minNonZerodisableIndexTimestamp - 1);
                  }
-             }
-             // Freeze time for table at min non-zero value of INDEX_DISABLE_TIMESTAMP
-             // This will keep the table consistent with index as the table has had one more
-             // batch applied to it.
-             if (minNonZerodisableIndexTimestamp == Long.MAX_VALUE) {
-                 builder.setMutationTime(currentTime);
-             } else {
-                 // Subtract one because we add one due to timestamp granularity in Windows
-                 builder.setMutationTime(minNonZerodisableIndexTimestamp - 1);
              }
 -
 -            if (table.getTimeStamp() != tableTimeStamp) {
 +            Pair<PTable, MetaDataProtos.MutationCode> pair = combineColumns(table, tenantId, schemaName, tableName, request.getClientTimestamp(), request.getClientVersion());
 +            table = pair.getFirst();
 +            if (table == null) {
 +                builder.setReturnCode(pair.getSecond());
 +                builder.setMutationTime(currentTime);
 +            }
 +            else if (table.getTimeStamp() != tableTimeStamp) {
                  builder.setTable(PTableImpl.toProto(table));
              }
              done.run(builder.build());
@@@ -1250,10 -1066,10 +1280,10 @@@
          return PTableImpl.makePTable(tenantId, schemaName, tableName, tableType, indexState, timeStamp, tableSeqNum,
                  pkName, saltBucketNum, columns, parentSchemaName, parentTableName, indexes, isImmutableRows, physicalTables, defaultFamilyName,
                  viewStatement, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType,
-                 rowKeyOrderOptimizable, transactional, updateCacheFrequency, baseColumnCount,
+                 rowKeyOrderOptimizable, transactionProvider, updateCacheFrequency, baseColumnCount,
                  indexDisableTimestamp, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, storageScheme, encodingScheme, cqCounter, useStatsForParallelization);
      }
 -    
 +
      private boolean isQualifierCounterKV(Cell kv) {
          int cmp =
                  Bytes.compareTo(kv.getQualifierArray(), kv.getQualifierOffset(),
@@@ -2146,12 -1998,16 +2183,16 @@@
              byte[] key =
                      parentTableName == null ? lockKey : SchemaUtil.getTableKey(tenantIdBytes,
                          schemaName, tableName);
- 
-             
+             Region region = env.getRegion();
+             MetaDataMutationResult result = checkTableKeyInRegion(key, region);
+             if (result != null) {
+                 done.run(MetaDataMutationResult.toProto(result));
+                 return;
+             }
              PTableType ptableType=PTableType.fromSerializedValue(tableType);
 -            long clientTimeStamp = MetaDataUtil.getClientTimeStamp(tableMetadata);
 +            long clientTimeStamp = MetaDataUtil.getClientTimeStamp(catalogMutations);
              byte[] cKey = SchemaUtil.getTableKey(tenantIdBytes, schemaName, tableName);
-             PTable loadedTable = loadTable(env, cKey, new ImmutableBytesPtr(cKey), clientTimeStamp, clientTimeStamp,
+             PTable loadedTable = getTable(env, cKey, new ImmutableBytesPtr(cKey), clientTimeStamp, clientTimeStamp,
                      request.getClientVersion());
              if (loadedTable == null) {
                  builder.setReturnCode(MetaDataProtos.MutationCode.TABLE_NOT_FOUND);
@@@ -2163,14 -2019,8 +2204,7 @@@
                      SchemaUtil.getTableName(schemaName, tableName),
                      TableName.valueOf(loadedTable.getPhysicalName().getBytes()),
                      getParentPhysicalTableName(loadedTable), ptableType,loadedTable.getIndexes());
- 
-             Region region = env.getRegion();
-             MetaDataMutationResult result = checkTableKeyInRegion(key, region);
-             if (result != null) {
-                 done.run(MetaDataMutationResult.toProto(result));
-                 return;
-             }
              List<RowLock> locks = Lists.newArrayList();
 -
              try {
                  acquireLock(region, lockKey, locks);
                  if (key != lockKey) {
@@@ -2746,82 -2490,75 +2781,58 @@@
          }
          return false;
      }
 -    
 -    private MetaDataMutationResult addColumnsAndTablePropertiesToChildViews(PTable basePhysicalTable, List<Mutation> tableMetadata, List<Mutation> mutationsForAddingColumnsToViews, byte[] schemaName, byte[] tableName,
 -            List<ImmutableBytesPtr> invalidateList, long clientTimeStamp, TableViewFinder childViewsResult,
 -            Region region, List<RowLock> locks, int clientVersion) throws IOException, SQLException {
 -        List<PutWithOrdinalPosition> columnPutsForBaseTable = Lists.newArrayListWithExpectedSize(tableMetadata.size());
 -        Map<TableProperty, Cell> tablePropertyCellMap = Maps.newHashMapWithExpectedSize(tableMetadata.size());
 -        // Isolate the puts relevant to adding columns. Also figure out what kind of columns are being added.
 +
 +
 +    private MetaDataMutationResult validateColumnForAddToBaseTable(PTable basePhysicalTable,
 +            List<Mutation> tableMetadata, byte[][] rowKeyMetaData,
 +            TableViewFinderResult childViewsResult, long clientTimeStamp, int clientVersion)
 +            throws IOException, SQLException {
 +        byte[] schemaName = rowKeyMetaData[SCHEMA_NAME_INDEX];
 +        byte[] tableName = rowKeyMetaData[TABLE_NAME_INDEX];
 +        List<PutWithOrdinalPosition> columnPutsForBaseTable =
 +                Lists.newArrayListWithExpectedSize(tableMetadata.size());
-         Map<TableProperty, Cell> tablePropertyCellMap =
-                 Maps.newHashMapWithExpectedSize(tableMetadata.size());
 +        // Isolate the puts relevant to adding columns. Also figure out what kind of columns are
 +        // being added.
          for (Mutation m : tableMetadata) {
              if (m instanceof Put) {
                  byte[][] rkmd = new byte[5][];
                  int pkCount = getVarChars(m.getRow(), rkmd);
                  // check if this put is for adding a column
 -                if (pkCount > COLUMN_NAME_INDEX
 -                        && rkmd[COLUMN_NAME_INDEX] != null && rkmd[COLUMN_NAME_INDEX].length > 0
 -                        && Bytes.compareTo(schemaName, rkmd[SCHEMA_NAME_INDEX]) == 0
 -                        && Bytes.compareTo(tableName, rkmd[TABLE_NAME_INDEX]) == 0) {
 -                    columnPutsForBaseTable.add(new PutWithOrdinalPosition((Put)m, getInteger((Put)m, TABLE_FAMILY_BYTES, ORDINAL_POSITION_BYTES)));
 -                }
 -                // check if the put is for a table property
 -                else if (pkCount <= COLUMN_NAME_INDEX
 +                if (pkCount > COLUMN_NAME_INDEX && rkmd[COLUMN_NAME_INDEX] != null
 +                        && rkmd[COLUMN_NAME_INDEX].length > 0
                          && Bytes.compareTo(schemaName, rkmd[SCHEMA_NAME_INDEX]) == 0
                          && Bytes.compareTo(tableName, rkmd[TABLE_NAME_INDEX]) == 0) {
 -                    for (Cell cell : m.getFamilyCellMap().get(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES)) {
 -                        for (TableProperty tableProp : TableProperty.values()) {
 -                            byte[] propNameBytes = Bytes.toBytes(tableProp.getPropertyName());
 -                            if (Bytes.compareTo(propNameBytes, 0, propNameBytes.length, cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength())==0
 -                                    && tableProp.isValidOnView() && tableProp.isMutable()) {
 -                                Cell tablePropCell = CellUtil.createCell(cell.getRow(), CellUtil.cloneFamily(cell),
 -                                    CellUtil.cloneQualifier(cell), cell.getTimestamp(), cell.getTypeByte(),
 -                                    CellUtil.cloneValue(cell));
 -                                tablePropertyCellMap.put(tableProp, tablePropCell);
 -                            }
 -                        }
 -                    }
 +                    columnPutsForBaseTable.add(new PutWithOrdinalPosition((Put) m,
 +                            getInteger((Put) m, TABLE_FAMILY_BYTES, ORDINAL_POSITION_BYTES)));
                  }
-                 // check if the put is for a table property
-                 else if (pkCount <= COLUMN_NAME_INDEX
-                         && Bytes.compareTo(schemaName, rkmd[SCHEMA_NAME_INDEX]) == 0
-                         && Bytes.compareTo(tableName, rkmd[TABLE_NAME_INDEX]) == 0) {
-                     for (Cell cell : m.getFamilyCellMap()
-                             .get(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES)) {
-                         for (TableProperty tableProp : TableProperty.values()) {
-                             byte[] propNameBytes = Bytes.toBytes(tableProp.getPropertyName());
-                             if (Bytes.compareTo(propNameBytes, 0, propNameBytes.length,
-                                 cell.getQualifierArray(), cell.getQualifierOffset(),
-                                 cell.getQualifierLength()) == 0 && tableProp.isValidOnView()
-                                     && tableProp.isMutable()) {
-                                 Cell tablePropCell =
-                                         CellUtil.createCell(cell.getRow(),
-                                             CellUtil.cloneFamily(cell),
-                                             CellUtil.cloneQualifier(cell), cell.getTimestamp(),
-                                             cell.getTypeByte(), CellUtil.cloneValue(cell));
-                                 tablePropertyCellMap.put(tableProp, tablePropCell);
-                             }
-                         }
-                     }
-                 }
              }
          }
 -        // Sort the puts by ordinal position 
 +        // Sort the puts by ordinal position
          Collections.sort(columnPutsForBaseTable);
 -        for (ViewInfo viewInfo : childViewsResult.getViewInfoList()) {
 -            short deltaNumPkColsSoFar = 0;
 -            short columnsAddedToView = 0;
 -            short columnsAddedToBaseTable = 0;
 -            byte[] tenantId = viewInfo.getTenantId();
 -            byte[] schema = viewInfo.getSchemaName();
 -            byte[] table = viewInfo.getViewName();
 -            byte[] viewKey = SchemaUtil.getTableKey(tenantId, schema, table);
 -            
 -            // lock the rows corresponding to views so that no other thread can modify the view meta-data
 -            RowLock viewRowLock = acquireLock(region, viewKey, locks);
 -            PTable view = doGetTable(viewKey, clientTimeStamp, viewRowLock, clientVersion);
 -            
 -            ColumnOrdinalPositionUpdateList ordinalPositionList = new ColumnOrdinalPositionUpdateList();
 +        for (TableInfo viewInfo : childViewsResult.getResults()) {
 +            byte[] viewKey = SchemaUtil.getTableKey(viewInfo.getTenantId(), viewInfo.getSchemaName(), viewInfo.getTableName());
 +            PTable view = doGetTable(viewKey, clientTimeStamp, clientVersion);
 +
 +            // add the new columns to the child view
              List<PColumn> viewPkCols = new ArrayList<>(view.getPKColumns());
              boolean addingExistingPkCol = false;
 -            int numCols = view.getColumns().size();
 -            // add the new columns to the child view
              for (PutWithOrdinalPosition p : columnPutsForBaseTable) {
 -                Put baseTableColumnPut = p.put;
 +                Put columnToBeAdded = p.put;
                  PColumn existingViewColumn = null;
                  byte[][] rkmd = new byte[5][];
 -                getVarChars(baseTableColumnPut.getRow(), rkmd);
 +                getVarChars(columnToBeAdded.getRow(), rkmd);
                  String columnName = Bytes.toString(rkmd[COLUMN_NAME_INDEX]);
 -                String columnFamily = rkmd[FAMILY_NAME_INDEX] == null ? null : Bytes.toString(rkmd[FAMILY_NAME_INDEX]);
 +                String columnFamily =
 +                        rkmd[FAMILY_NAME_INDEX] == null ? null
 +                                : Bytes.toString(rkmd[FAMILY_NAME_INDEX]);
                  try {
 -                    existingViewColumn = columnFamily == null ? view.getColumnForColumnName(columnName) : view.getColumnFamily(
 -                            columnFamily).getPColumnForColumnName(columnName);
 +                    existingViewColumn =
 +                            columnFamily == null ? view.getColumnForColumnName(columnName)
 +                                    : view.getColumnFamily(columnFamily)
 +                                            .getPColumnForColumnName(columnName);
                  } catch (ColumnFamilyNotFoundException e) {
 -                    // ignore since it means that the column family is not present for the column to be added.
 +                    // ignore since it means that the column family is not present for the column to
 +                    // be added.
                  } catch (ColumnNotFoundException e) {
                      // ignore since it means the column is not present in the view
                  }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
index b46894e,8119c6e..e088be2
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
@@@ -17112,51 -17193,52 +17202,52 @@@ public final class MetaDataProtos 
        "cheRequest\022\025\n\rclientVersion\030\001 \001(\005\"*\n\022Cle" +
        "arCacheResponse\022\024\n\014unfreedBytes\030\001 \001(\003\"*\n" +
        "\021GetVersionRequest\022\025\n\rclientVersion\030\001 \001(" +
-       "\005\"%\n\022GetVersionResponse\022\017\n\007version\030\001 \002(\003" +
-       "\"\205\001\n\032ClearTableFromCacheRequest\022\020\n\010tenan",
-       "tId\030\001 \002(\014\022\022\n\nschemaName\030\002 \002(\014\022\021\n\ttableNa" +
-       "me\030\003 \002(\014\022\027\n\017clientTimestamp\030\004 \002(\003\022\025\n\rcli" +
-       "entVersion\030\005 \001(\005\"\035\n\033ClearTableFromCacheR" +
-       "esponse*\207\005\n\014MutationCode\022\030\n\024TABLE_ALREAD" +
-       "Y_EXISTS\020\000\022\023\n\017TABLE_NOT_FOUND\020\001\022\024\n\020COLUM" +
-       "N_NOT_FOUND\020\002\022\031\n\025COLUMN_ALREADY_EXISTS\020\003" +
-       "\022\035\n\031CONCURRENT_TABLE_MUTATION\020\004\022\027\n\023TABLE" +
-       "_NOT_IN_REGION\020\005\022\025\n\021NEWER_TABLE_FOUND\020\006\022" +
-       "\034\n\030UNALLOWED_TABLE_MUTATION\020\007\022\021\n\rNO_PK_C" +
-       "OLUMNS\020\010\022\032\n\026PARENT_TABLE_NOT_FOUND\020\t\022\033\n\027",
-       "FUNCTION_ALREADY_EXISTS\020\n\022\026\n\022FUNCTION_NO" +
-       "T_FOUND\020\013\022\030\n\024NEWER_FUNCTION_FOUND\020\014\022\032\n\026F" +
-       "UNCTION_NOT_IN_REGION\020\r\022\031\n\025SCHEMA_ALREAD" +
-       "Y_EXISTS\020\016\022\026\n\022NEWER_SCHEMA_FOUND\020\017\022\024\n\020SC" +
-       "HEMA_NOT_FOUND\020\020\022\030\n\024SCHEMA_NOT_IN_REGION" +
-       "\020\021\022\032\n\026TABLES_EXIST_ON_SCHEMA\020\022\022\035\n\031UNALLO" +
-       "WED_SCHEMA_MUTATION\020\023\022%\n!AUTO_PARTITION_" +
-       "SEQUENCE_NOT_FOUND\020\024\022#\n\037CANNOT_COERCE_AU" +
-       "TO_PARTITION_ID\020\025\022\024\n\020TOO_MANY_INDEXES\020\026\022" +
-       "\020\n\014INVALID_VIEW\020\0272\345\006\n\017MetaDataService\022/\n",
-       "\010getTable\022\020.GetTableRequest\032\021.MetaDataRe" +
-       "sponse\0227\n\014getFunctions\022\024.GetFunctionsReq" +
-       "uest\032\021.MetaDataResponse\0221\n\tgetSchema\022\021.G" +
-       "etSchemaRequest\032\021.MetaDataResponse\0225\n\013cr" +
-       "eateTable\022\023.CreateTableRequest\032\021.MetaDat" +
-       "aResponse\022;\n\016createFunction\022\026.CreateFunc" +
-       "tionRequest\032\021.MetaDataResponse\0227\n\014create" +
-       "Schema\022\024.CreateSchemaRequest\032\021.MetaDataR" +
-       "esponse\0221\n\tdropTable\022\021.DropTableRequest\032" +
-       "\021.MetaDataResponse\0223\n\ndropSchema\022\022.DropS",
-       "chemaRequest\032\021.MetaDataResponse\0227\n\014dropF" +
-       "unction\022\024.DropFunctionRequest\032\021.MetaData" +
-       "Response\0221\n\taddColumn\022\021.AddColumnRequest" +
-       "\032\021.MetaDataResponse\0223\n\ndropColumn\022\022.Drop" +
-       "ColumnRequest\032\021.MetaDataResponse\022?\n\020upda" +
-       "teIndexState\022\030.UpdateIndexStateRequest\032\021" +
-       ".MetaDataResponse\0225\n\nclearCache\022\022.ClearC" +
-       "acheRequest\032\023.ClearCacheResponse\0225\n\ngetV" +
-       "ersion\022\022.GetVersionRequest\032\023.GetVersionR" +
-       "esponse\022P\n\023clearTableFromCache\022\033.ClearTa",
-       "bleFromCacheRequest\032\034.ClearTableFromCach" +
-       "eResponseBB\n(org.apache.phoenix.coproces" +
-       "sor.generatedB\016MetaDataProtosH\001\210\001\001\240\001\001"
+       "\005\"E\n\022GetVersionResponse\022\017\n\007version\030\001 \002(\003" +
+       "\022\036\n\026systemCatalogTimestamp\030\002 \001(\003\"\205\001\n\032Cle",
+       "arTableFromCacheRequest\022\020\n\010tenantId\030\001 \002(" +
+       "\014\022\022\n\nschemaName\030\002 \002(\014\022\021\n\ttableName\030\003 \002(\014" +
+       "\022\027\n\017clientTimestamp\030\004 \002(\003\022\025\n\rclientVersi" +
+       "on\030\005 \001(\005\"\035\n\033ClearTableFromCacheResponse*" +
 -      "\365\004\n\014MutationCode\022\030\n\024TABLE_ALREADY_EXISTS" +
++      "\207\005\n\014MutationCode\022\030\n\024TABLE_ALREADY_EXISTS" +
+       "\020\000\022\023\n\017TABLE_NOT_FOUND\020\001\022\024\n\020COLUMN_NOT_FO" +
+       "UND\020\002\022\031\n\025COLUMN_ALREADY_EXISTS\020\003\022\035\n\031CONC" +
+       "URRENT_TABLE_MUTATION\020\004\022\027\n\023TABLE_NOT_IN_" +
+       "REGION\020\005\022\025\n\021NEWER_TABLE_FOUND\020\006\022\034\n\030UNALL" +
+       "OWED_TABLE_MUTATION\020\007\022\021\n\rNO_PK_COLUMNS\020\010",
+       "\022\032\n\026PARENT_TABLE_NOT_FOUND\020\t\022\033\n\027FUNCTION" +
+       "_ALREADY_EXISTS\020\n\022\026\n\022FUNCTION_NOT_FOUND\020" +
+       "\013\022\030\n\024NEWER_FUNCTION_FOUND\020\014\022\032\n\026FUNCTION_" +
+       "NOT_IN_REGION\020\r\022\031\n\025SCHEMA_ALREADY_EXISTS" +
+       "\020\016\022\026\n\022NEWER_SCHEMA_FOUND\020\017\022\024\n\020SCHEMA_NOT" +
+       "_FOUND\020\020\022\030\n\024SCHEMA_NOT_IN_REGION\020\021\022\032\n\026TA" +
+       "BLES_EXIST_ON_SCHEMA\020\022\022\035\n\031UNALLOWED_SCHE" +
+       "MA_MUTATION\020\023\022%\n!AUTO_PARTITION_SEQUENCE" +
+       "_NOT_FOUND\020\024\022#\n\037CANNOT_COERCE_AUTO_PARTI" +
 -      "TION_ID\020\025\022\024\n\020TOO_MANY_INDEXES\020\0262\345\006\n\017Meta",
 -      "DataService\022/\n\010getTable\022\020.GetTableReques" +
 -      "t\032\021.MetaDataResponse\0227\n\014getFunctions\022\024.G" +
 -      "etFunctionsRequest\032\021.MetaDataResponse\0221\n" +
 -      "\tgetSchema\022\021.GetSchemaRequest\032\021.MetaData" +
 -      "Response\0225\n\013createTable\022\023.CreateTableReq" +
 -      "uest\032\021.MetaDataResponse\022;\n\016createFunctio" +
 -      "n\022\026.CreateFunctionRequest\032\021.MetaDataResp" +
 -      "onse\0227\n\014createSchema\022\024.CreateSchemaReque" +
 -      "st\032\021.MetaDataResponse\0221\n\tdropTable\022\021.Dro" +
 -      "pTableRequest\032\021.MetaDataResponse\0223\n\ndrop",
 -      "Schema\022\022.DropSchemaRequest\032\021.MetaDataRes" +
 -      "ponse\0227\n\014dropFunction\022\024.DropFunctionRequ" +
 -      "est\032\021.MetaDataResponse\0221\n\taddColumn\022\021.Ad" +
 -      "dColumnRequest\032\021.MetaDataResponse\0223\n\ndro" +
 -      "pColumn\022\022.DropColumnRequest\032\021.MetaDataRe" +
 -      "sponse\022?\n\020updateIndexState\022\030.UpdateIndex" +
 -      "StateRequest\032\021.MetaDataResponse\0225\n\nclear" +
 -      "Cache\022\022.ClearCacheRequest\032\023.ClearCacheRe" +
 -      "sponse\0225\n\ngetVersion\022\022.GetVersionRequest" +
 -      "\032\023.GetVersionResponse\022P\n\023clearTableFromC",
 -      "ache\022\033.ClearTableFromCacheRequest\032\034.Clea" +
 -      "rTableFromCacheResponseBB\n(org.apache.ph" +
 -      "oenix.coprocessor.generatedB\016MetaDataPro" +
 -      "tosH\001\210\001\001\240\001\001"
++      "TION_ID\020\025\022\024\n\020TOO_MANY_INDEXES\020\026\022\020\n\014INVAL",
++      "ID_VIEW\020\0272\345\006\n\017MetaDataService\022/\n\010getTabl" +
++      "e\022\020.GetTableRequest\032\021.MetaDataResponse\0227" +
++      "\n\014getFunctions\022\024.GetFunctionsRequest\032\021.M" +
++      "etaDataResponse\0221\n\tgetSchema\022\021.GetSchema" +
++      "Request\032\021.MetaDataResponse\0225\n\013createTabl" +
++      "e\022\023.CreateTableRequest\032\021.MetaDataRespons" +
++      "e\022;\n\016createFunction\022\026.CreateFunctionRequ" +
++      "est\032\021.MetaDataResponse\0227\n\014createSchema\022\024" +
++      ".CreateSchemaRequest\032\021.MetaDataResponse\022" +
++      "1\n\tdropTable\022\021.DropTableRequest\032\021.MetaDa",
++      "taResponse\0223\n\ndropSchema\022\022.DropSchemaReq" +
++      "uest\032\021.MetaDataResponse\0227\n\014dropFunction\022" +
++      "\024.DropFunctionRequest\032\021.MetaDataResponse" +
++      "\0221\n\taddColumn\022\021.AddColumnRequest\032\021.MetaD" +
++      "ataResponse\0223\n\ndropColumn\022\022.DropColumnRe" +
++      "quest\032\021.MetaDataResponse\022?\n\020updateIndexS" +
++      "tate\022\030.UpdateIndexStateRequest\032\021.MetaDat" +
++      "aResponse\0225\n\nclearCache\022\022.ClearCacheRequ" +
++      "est\032\023.ClearCacheResponse\0225\n\ngetVersion\022\022" +
++      ".GetVersionRequest\032\023.GetVersionResponse\022",
++      "P\n\023clearTableFromCache\022\033.ClearTableFromC" +
++      "acheRequest\032\034.ClearTableFromCacheRespons" +
++      "eBB\n(org.apache.phoenix.coprocessor.gene" +
++      "ratedB\016MetaDataProtosH\001\210\001\001\240\001\001"
      };
      com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
        new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
index 12bfcf9,8d500e8..9b8f45e
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
@@@ -8764,39 -8677,39 +8857,40 @@@ public final class PTableProtos 
        "\001(\014\022\026\n\016viewReferenced\030\013 \001(\010\022\022\n\nexpressio" +
        "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\022\021\n\tisDyn" +
        "amic\030\016 \001(\010\022\034\n\024columnQualifierBytes\030\017 \001(\014" +
 -      "\"\232\001\n\013PTableStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030",
 -      "\002 \003(\014\022\033\n\023guidePostsByteCount\030\003 \001(\003\022\025\n\rke" +
 -      "yBytesCount\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001" +
 -      "(\005\022!\n\013pGuidePosts\030\006 \001(\0132\014.PGuidePosts\"\255\007" +
 -      "\n\006PTable\022\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tab" +
 -      "leNameBytes\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.P" +
 -      "TableType\022\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenc" +
 -      "eNumber\030\005 \002(\003\022\021\n\ttimeStamp\030\006 \002(\003\022\023\n\013pkNa" +
 -      "meBytes\030\007 \001(\014\022\021\n\tbucketNum\030\010 \002(\005\022\031\n\007colu" +
 -      "mns\030\t \003(\0132\010.PColumn\022\030\n\007indexes\030\n \003(\0132\007.P" +
 -      "Table\022\027\n\017isImmutableRows\030\013 \002(\010\022\032\n\022dataTa",
 -      "bleNameBytes\030\r \001(\014\022\031\n\021defaultFamilyName\030" +
 -      "\016 \001(\014\022\022\n\ndisableWAL\030\017 \002(\010\022\023\n\013multiTenant" +
 -      "\030\020 \002(\010\022\020\n\010viewType\030\021 \001(\014\022\025\n\rviewStatemen" +
 -      "t\030\022 \001(\014\022\025\n\rphysicalNames\030\023 \003(\014\022\020\n\010tenant" +
 -      "Id\030\024 \001(\014\022\023\n\013viewIndexId\030\025 \001(\005\022\021\n\tindexTy" +
 -      "pe\030\026 \001(\014\022\026\n\016statsTimeStamp\030\027 \001(\003\022\022\n\nstor" +
 -      "eNulls\030\030 \001(\010\022\027\n\017baseColumnCount\030\031 \001(\005\022\036\n" +
 -      "\026rowKeyOrderOptimizable\030\032 \001(\010\022\025\n\rtransac" +
 -      "tional\030\033 \001(\010\022\034\n\024updateCacheFrequency\030\034 \001" +
 -      "(\003\022\035\n\025indexDisableTimestamp\030\035 \001(\003\022\031\n\021isN",
 -      "amespaceMapped\030\036 \001(\010\022\034\n\024autoParititonSeq" +
 -      "Name\030\037 \001(\t\022\032\n\022isAppendOnlySchema\030  \001(\010\022\027" +
 -      "\n\017parentNameBytes\030! \001(\014\022\025\n\rstorageScheme" +
 -      "\030\" \001(\014\022\026\n\016encodingScheme\030# \001(\014\022,\n\021encode" +
 -      "dCQCounters\030$ \003(\0132\021.EncodedCQCounter\022\"\n\032" +
 -      "useStatsForParallelization\030% \001(\010\022\033\n\023tran" +
 -      "sactionProvider\030& \001(\005\"6\n\020EncodedCQCounte" +
 -      "r\022\021\n\tcolFamily\030\001 \002(\t\022\017\n\007counter\030\002 \002(\005*A\n" +
 -      "\nPTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIE" +
 -      "W\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.p",
 -      "hoenix.coprocessor.generatedB\014PTableProt" +
 -      "osH\001\210\001\001\240\001\001"
 +      "\022\021\n\ttimestamp\030\020 \001(\003\022\026\n\007derived\030\021 \001(\010:\005fa",
 +      "lse\"\232\001\n\013PTableStats\022\013\n\003key\030\001 \002(\014\022\016\n\006valu" +
 +      "es\030\002 \003(\014\022\033\n\023guidePostsByteCount\030\003 \001(\003\022\025\n" +
 +      "\rkeyBytesCount\030\004 \001(\003\022\027\n\017guidePostsCount\030" +
 +      "\005 \001(\005\022!\n\013pGuidePosts\030\006 \001(\0132\014.PGuidePosts" +
-       "\"\220\007\n\006PTable\022\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016" +
++      "\"\255\007\n\006PTable\022\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016" +
 +      "tableNameBytes\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162" +
 +      "\013.PTableType\022\022\n\nindexState\030\004 \001(\t\022\026\n\016sequ" +
 +      "enceNumber\030\005 \002(\003\022\021\n\ttimeStamp\030\006 \002(\003\022\023\n\013p" +
 +      "kNameBytes\030\007 \001(\014\022\021\n\tbucketNum\030\010 \002(\005\022\031\n\007c" +
 +      "olumns\030\t \003(\0132\010.PColumn\022\030\n\007indexes\030\n \003(\0132",
 +      "\007.PTable\022\027\n\017isImmutableRows\030\013 \002(\010\022\032\n\022dat" +
 +      "aTableNameBytes\030\r \001(\014\022\031\n\021defaultFamilyNa" +
 +      "me\030\016 \001(\014\022\022\n\ndisableWAL\030\017 \002(\010\022\023\n\013multiTen" +
 +      "ant\030\020 \002(\010\022\020\n\010viewType\030\021 \001(\014\022\025\n\rviewState" +
 +      "ment\030\022 \001(\014\022\025\n\rphysicalNames\030\023 \003(\014\022\020\n\010ten" +
 +      "antId\030\024 \001(\014\022\023\n\013viewIndexId\030\025 \001(\005\022\021\n\tinde" +
 +      "xType\030\026 \001(\014\022\026\n\016statsTimeStamp\030\027 \001(\003\022\022\n\ns" +
 +      "toreNulls\030\030 \001(\010\022\027\n\017baseColumnCount\030\031 \001(\005" +
 +      "\022\036\n\026rowKeyOrderOptimizable\030\032 \001(\010\022\025\n\rtran" +
 +      "sactional\030\033 \001(\010\022\034\n\024updateCacheFrequency\030",
 +      "\034 \001(\003\022\035\n\025indexDisableTimestamp\030\035 \001(\003\022\031\n\021" +
 +      "isNamespaceMapped\030\036 \001(\010\022\034\n\024autoParititon" +
 +      "SeqName\030\037 \001(\t\022\032\n\022isAppendOnlySchema\030  \001(" +
 +      "\010\022\027\n\017parentNameBytes\030! \001(\014\022\025\n\rstorageSch" +
 +      "eme\030\" \001(\014\022\026\n\016encodingScheme\030# \001(\014\022,\n\021enc" +
 +      "odedCQCounters\030$ \003(\0132\021.EncodedCQCounter\022" +
-       "\"\n\032useStatsForParallelization\030% \001(\010\"6\n\020E" +
-       "ncodedCQCounter\022\021\n\tcolFamily\030\001 \002(\t\022\017\n\007co" +
-       "unter\030\002 \002(\005*A\n\nPTableType\022\n\n\006SYSTEM\020\000\022\010\n" +
-       "\004USER\020\001\022\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@",
-       "\n(org.apache.phoenix.coprocessor.generat" +
-       "edB\014PTableProtosH\001\210\001\001\240\001\001"
++      "\"\n\032useStatsForParallelization\030% \001(\010\022\033\n\023t" +
++      "ransactionProvider\030& \001(\005\"6\n\020EncodedCQCou" +
++      "nter\022\021\n\tcolFamily\030\001 \002(\t\022\017\n\007counter\030\002 \002(\005" +
++      "*A\n\nPTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004",
++      "VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.apach" +
++      "e.phoenix.coprocessor.generatedB\014PTableP" +
++      "rotosH\001\210\001\001\240\001\001"
      };
      com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
        new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index 3a42584,e9e209b..d29ef66
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@@ -294,9 -293,11 +293,13 @@@ public enum SQLExceptionCode 
      
      SEQUENCE_NOT_CASTABLE_TO_AUTO_PARTITION_ID_COLUMN(1086, "44A17", "Sequence Value not castable to auto-partition id column"),
      CANNOT_COERCE_AUTO_PARTITION_ID(1087, "44A18", "Auto-partition id cannot be coerced"),
 +    
      CANNOT_CREATE_INDEX_ON_MUTABLE_TABLE_WITH_ROWTIMESTAMP(1088, "44A19", "Cannot create an index on a mutable table that has a ROW_TIMESTAMP column."),
-     INVALID_VIEW(1089, "44A20", "View is invalid as a required column was dropped"),
+     UNKNOWN_TRANSACTION_PROVIDER(1089,"44A20", "Unknown TRANSACTION_PROVIDER: "),
+     CANNOT_START_TXN_IF_TXN_DISABLED(1091, "44A22", "Cannot start transaction if transactions are disabled."),
+     CANNOT_MIX_TXN_PROVIDERS(1092, "44A23", "Cannot mix transaction providers: "),
+     CANNOT_ALTER_TABLE_FROM_NON_TXN_TO_TXNL(1093, "44A24", "Cannot alter table from non transactional to transactional for "),
++    INVALID_VIEW(1094, "44A25", "View is invalid as a required column was dropped"),
  
      /** Sequence related */
      SEQUENCE_ALREADY_EXIST(1200, "42Z00", "Sequence already exists.", new Factory() {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index fbb5a42,d56628a..7fbaef1
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@@ -368,12 -347,25 +375,30 @@@ public class PhoenixDatabaseMetaData im
      public static final byte[] COLUMN_QUALIFIER_COUNTER_BYTES = Bytes.toBytes(COLUMN_QUALIFIER_COUNTER);
      public static final String USE_STATS_FOR_PARALLELIZATION = "USE_STATS_FOR_PARALLELIZATION";
      public static final byte[] USE_STATS_FOR_PARALLELIZATION_BYTES = Bytes.toBytes(USE_STATS_FOR_PARALLELIZATION);
 +    
 +    public static final String SYSTEM_CHILD_LINK_TABLE = "CHILD_LINK";
 +    public static final String SYSTEM_CHILD_LINK_NAME = SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_CHILD_LINK_TABLE);
 +    public static final byte[] SYSTEM_CHILD_LINK_NAME_BYTES = Bytes.toBytes(SYSTEM_CHILD_LINK_NAME);
 +    public static final TableName SYSTEM_LINK_HBASE_TABLE_NAME = TableName.valueOf(SYSTEM_CHILD_LINK_NAME);
  
+     
+     //SYSTEM:LOG
+     public static final String SYSTEM_LOG_TABLE = "LOG";
+     public static final String QUERY_ID = "QUERY_ID";
+     public static final String USER = "USER";
+     public static final String CLIENT_IP = "CLIENT_IP";
+     public static final String QUERY = "QUERY";
+     public static final String EXPLAIN_PLAN = "EXPLAIN_PLAN";
+     public static final String TOTAL_EXECUTION_TIME = "TOTAL_EXECUTION_TIME";
+     public static final String NO_OF_RESULTS_ITERATED = "NO_OF_RESULTS_ITERATED";
+     public static final String QUERY_STATUS = "QUERY_STATUS";
+     public static final String EXCEPTION_TRACE = "EXCEPTION_TRACE";
+     public static final String GLOBAL_SCAN_DETAILS = "GLOBAL_SCAN_DETAILS";
+     public static final String SCAN_METRICS_JSON = "SCAN_METRICS_JSON";
+     public static final String START_TIME = "START_TIME";
+     public static final String BIND_PARAMETERS = "BIND_PARAMETERS";
+             
+     
      PhoenixDatabaseMetaData(PhoenixConnection connection) throws SQLException {
          this.emptyResultSet = new PhoenixResultSet(ResultIterator.EMPTY_ITERATOR, RowProjector.EMPTY_PROJECTOR, new StatementContext(new PhoenixStatement(connection), false));
          this.connection = connection;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index cb47a73,25b9fb0..3e9a597
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@@ -515,9 -531,9 +531,9 @@@ public class PhoenixStatement implement
      public static final String EXPLAIN_PLAN_BYTES_ESTIMATE_COLUMN_ALIAS = "EST_BYTES_READ";
      private static final PColumnImpl EXPLAIN_PLAN_BYTES_ESTIMATE_COLUMN =
              new PColumnImpl(PNameFactory.newName(EXPLAIN_PLAN_BYTES_ESTIMATE),
-                     PNameFactory.newName(EXPLAIN_PLAN_FAMILY), PLong.INSTANCE, null, null, false, 1,
+                     PNameFactory.newName(EXPLAIN_PLAN_FAMILY), PLong.INSTANCE, null, null, true, 1,
                      SortOrder.getDefault(), 0, null, false, null, false, false,
 -                    EXPLAIN_PLAN_BYTES_ESTIMATE);
 +                    EXPLAIN_PLAN_BYTES_ESTIMATE, 0, false);
  
      private static final String EXPLAIN_PLAN_ROWS_ESTIMATE_COLUMN_NAME = "RowsEstimate";
      private static final byte[] EXPLAIN_PLAN_ROWS_ESTIMATE =
@@@ -525,9 -541,9 +541,9 @@@
      public static final String EXPLAIN_PLAN_ROWS_COLUMN_ALIAS = "EST_ROWS_READ";
      private static final PColumnImpl EXPLAIN_PLAN_ROWS_ESTIMATE_COLUMN =
              new PColumnImpl(PNameFactory.newName(EXPLAIN_PLAN_ROWS_ESTIMATE),
-                     PNameFactory.newName(EXPLAIN_PLAN_FAMILY), PLong.INSTANCE, null, null, false, 2,
+                     PNameFactory.newName(EXPLAIN_PLAN_FAMILY), PLong.INSTANCE, null, null, true, 2,
                      SortOrder.getDefault(), 0, null, false, null, false, false,
 -                    EXPLAIN_PLAN_ROWS_ESTIMATE);
 +                    EXPLAIN_PLAN_ROWS_ESTIMATE, 0, false);
  
      private static final String EXPLAIN_PLAN_ESTIMATE_INFO_TS_COLUMN_NAME = "EstimateInfoTS";
      private static final byte[] EXPLAIN_PLAN_ESTIMATE_INFO_TS =
@@@ -535,9 -551,9 +551,9 @@@
      public static final String EXPLAIN_PLAN_ESTIMATE_INFO_TS_COLUMN_ALIAS = "EST_INFO_TS";
      private static final PColumnImpl EXPLAIN_PLAN_ESTIMATE_INFO_TS_COLUMN =
              new PColumnImpl(PNameFactory.newName(EXPLAIN_PLAN_ESTIMATE_INFO_TS),
-                 PNameFactory.newName(EXPLAIN_PLAN_FAMILY), PLong.INSTANCE, null, null, false, 3,
+                 PNameFactory.newName(EXPLAIN_PLAN_FAMILY), PLong.INSTANCE, null, null, true, 3,
                  SortOrder.getDefault(), 0, null, false, null, false, false,
 -                EXPLAIN_PLAN_ESTIMATE_INFO_TS);
 +                EXPLAIN_PLAN_ESTIMATE_INFO_TS, 0, false);
  
      private static final RowProjector EXPLAIN_PLAN_ROW_PROJECTOR_WITH_BYTE_ROW_ESTIMATES =
              new RowProjector(Arrays

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d615dc0c/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 8bd06bc,8866ced..c453e13
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@@ -169,7 -170,8 +172,8 @@@ import org.apache.phoenix.exception.Ret
  import org.apache.phoenix.exception.SQLExceptionCode;
  import org.apache.phoenix.exception.SQLExceptionInfo;
  import org.apache.phoenix.exception.UpgradeInProgressException;
 -import org.apache.phoenix.exception.UpgradeRequiredException;
  import org.apache.phoenix.exception.UpgradeNotRequiredException;
++import org.apache.phoenix.exception.UpgradeRequiredException;
  import org.apache.phoenix.execute.MutationState;
  import org.apache.phoenix.hbase.index.IndexRegionSplitPolicy;
  import org.apache.phoenix.hbase.index.Indexer;
@@@ -2594,15 -2685,14 +2687,17 @@@ public class ConnectionQueryServicesImp
              metaConnection.createStatement().execute(QueryConstants.CREATE_STATS_TABLE_METADATA);
          } catch (TableAlreadyExistsException ignore) {}
          try {
-             metaConnection.createStatement().execute(QueryConstants.CREATE_FUNCTION_METADATA);
+             metaConnection.createStatement().execute(getFunctionTableDDL());
+         } catch (TableAlreadyExistsException ignore) {}
+         try {
+             metaConnection.createStatement().execute(getLogTableDDL());
          } catch (TableAlreadyExistsException ignore) {}
 +        try {
 +            metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_CHILD_LINK_METADATA);
 +        } catch (TableAlreadyExistsException e) {}
- 
          // Catch the IOException to log the error message and then bubble it up for the client to retry.
          try {
-             createSysMutexTable(hbaseAdmin, ConnectionQueryServicesImpl.this.getProps());
+             createSysMutexTableIfNotExists(hbaseAdmin);
          } catch (IOException exception) {
              logger.error("Failed to created SYSMUTEX table. Upgrade or migration is not possible without it. Please retry.");
              throw exception;
@@@ -2610,6 -2700,265 +2705,270 @@@
      }
  
      /**
+      * Create an entry for the SYSTEM namespace in the SYSCAT table in case namespace mapping is enabled and system table
+      * to system namespace mapping is also enabled. If not enabled, this method returns immediately without doing anything
+      * @param metaConnection
+      * @throws SQLException
+      */
+     private void createSchemaIfNotExistsSystemNSMappingEnabled(PhoenixConnection metaConnection) throws SQLException {
+         // HBase Namespace SYSTEM is assumed to be already created inside {@link ensureTableCreated(byte[], PTableType,
+         // Map<String, Object>, List<Pair<byte[], Map<String, Object>>>, byte[][], boolean, boolean, boolean)}.
+         // This statement will create an entry for the SYSTEM namespace in the SYSCAT table, so that GRANT/REVOKE
+         // commands can work with SYSTEM Namespace. (See PHOENIX-4227 https://issues.apache.org/jira/browse/PHOENIX-4227)
+         if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM,
+           ConnectionQueryServicesImpl.this.getProps())) {
+             try {
+                 metaConnection.createStatement().execute("CREATE SCHEMA IF NOT EXISTS "
+                   + PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA);
+             } catch (NewerSchemaAlreadyExistsException e) {
+                 // Older clients with appropriate perms may try getting a new connection
+                 // This results in NewerSchemaAlreadyExistsException, so we can safely ignore it here
+             } catch (PhoenixIOException e) {
+                 if (!Iterables.isEmpty(Iterables.filter(Throwables.getCausalChain(e), AccessDeniedException.class))) {
+                     // Ignore ADE
+                 } else {
+                     throw e;
+                 }
+             }
+         }
+     }
+ 
+     /**
+      * Upgrade the SYSCAT schema if required
+      * @param metaConnection
+      * @param currentServerSideTableTimeStamp
+      * @return Phoenix connection object
+      * @throws SQLException
+      * @throws IOException
+      * @throws TimeoutException
+      * @throws InterruptedException
+      */
+     // Available for testing
+     protected PhoenixConnection upgradeSystemCatalogIfRequired(PhoenixConnection metaConnection,
+       long currentServerSideTableTimeStamp) throws SQLException, IOException, TimeoutException, InterruptedException {
+         String columnsToAdd = "";
+         // This will occur if we have an older SYSTEM.CATALOG and we need to update it to
+         // include any new columns we've added.
+         if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0) {
+             // We know that we always need to add the STORE_NULLS column for 4.3 release
+             columnsToAdd = addColumn(columnsToAdd, PhoenixDatabaseMetaData.STORE_NULLS
+               + " " + PBoolean.INSTANCE.getSqlTypeName());
+             try (HBaseAdmin admin = getAdmin()) {
+                 HTableDescriptor[] localIndexTables = admin
+                   .listTables(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX + ".*");
+                 for (HTableDescriptor table : localIndexTables) {
+                     if (table.getValue(MetaDataUtil.PARENT_TABLE_KEY) == null
+                       && table.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_NAME) != null) {
+                         table.setValue(MetaDataUtil.PARENT_TABLE_KEY,
+                           MetaDataUtil.getLocalIndexUserTableName(table.getNameAsString()));
+                         // Explicitly disable, modify and enable the table to ensure
+                         // co-location of data and index regions. If we just modify the
+                         // table descriptor when online schema change enabled may reopen
+                         // the region in same region server instead of following data region.
+                         admin.disableTable(table.getTableName());
+                         admin.modifyTable(table.getTableName(), table);
+                         admin.enableTable(table.getTableName());
+                     }
+                 }
+             }
+         }
+ 
+         // If the server side schema is before MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0 then
+         // we need to add INDEX_TYPE and INDEX_DISABLE_TIMESTAMP columns too.
+         // TODO: Once https://issues.apache.org/jira/browse/PHOENIX-1614 is fixed,
+         // we should just have a ALTER TABLE ADD IF NOT EXISTS statement with all
+         // the column names that have been added to SYSTEM.CATALOG since 4.0.
+         if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0) {
+             columnsToAdd = addColumn(columnsToAdd, PhoenixDatabaseMetaData.INDEX_TYPE + " "
+               + PUnsignedTinyint.INSTANCE.getSqlTypeName() + ", "
+               + PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP + " "
+               + PLong.INSTANCE.getSqlTypeName());
+         }
+ 
+         // If we have some new columns from 4.1-4.3 to add, add them now.
+         if (!columnsToAdd.isEmpty()) {
+             // Ugh..need to assign to another local variable to keep eclipse happy.
+             PhoenixConnection newMetaConnection = addColumnsIfNotExists(metaConnection,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0, columnsToAdd);
+             metaConnection = newMetaConnection;
+         }
+ 
+         if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0) {
+             columnsToAdd = PhoenixDatabaseMetaData.BASE_COLUMN_COUNT + " "
+               + PInteger.INSTANCE.getSqlTypeName();
+             try {
+                 metaConnection = addColumn(metaConnection,
+                   PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+                   MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0, columnsToAdd,
+                   false);
+                 upgradeTo4_5_0(metaConnection);
+             } catch (ColumnAlreadyExistsException ignored) {
+                 /*
+                  * Upgrade to 4.5 is a slightly special case. We use the fact that the
+                  * column BASE_COLUMN_COUNT is already part of the meta-data schema as the
+                  * signal that the server side upgrade has finished or is in progress.
+                  */
+                 logger.debug("No need to run 4.5 upgrade");
+             }
+             Properties p = PropertiesUtil.deepCopy(metaConnection.getClientInfo());
+             p.remove(PhoenixRuntime.CURRENT_SCN_ATTRIB);
+             p.remove(PhoenixRuntime.TENANT_ID_ATTRIB);
+             PhoenixConnection conn = new PhoenixConnection(
+               ConnectionQueryServicesImpl.this, metaConnection.getURL(), p,
+               metaConnection.getMetaDataCache());
+             try {
+                 List<String> tablesNeedingUpgrade = UpgradeUtil
+                   .getPhysicalTablesWithDescRowKey(conn);
+                 if (!tablesNeedingUpgrade.isEmpty()) {
+                     logger.warn("The following tables require upgrade due to a bug causing the row key to be incorrect for descending columns and ascending BINARY columns (PHOENIX-2067 and PHOENIX-2120):\n"
+                       + Joiner.on(' ').join(tablesNeedingUpgrade)
+                       + "\nTo upgrade issue the \"bin/psql.py -u\" command.");
+                 }
+                 List<String> unsupportedTables = UpgradeUtil
+                   .getPhysicalTablesWithDescVarbinaryRowKey(conn);
+                 if (!unsupportedTables.isEmpty()) {
+                     logger.warn("The following tables use an unsupported VARBINARY DESC construct and need to be changed:\n"
+                       + Joiner.on(' ').join(unsupportedTables));
+                 }
+             } catch (Exception ex) {
+                 logger.error(
+                   "Unable to determine tables requiring upgrade due to PHOENIX-2067",
+                   ex);
+             } finally {
+                 conn.close();
+             }
+         }
+         // Add these columns one at a time, each with different timestamps so that if folks
+         // have
+         // run the upgrade code already for a snapshot, we'll still enter this block (and do
+         // the
+         // parts we haven't yet done).
+         if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0) {
+             columnsToAdd = PhoenixDatabaseMetaData.IS_ROW_TIMESTAMP + " "
+               + PBoolean.INSTANCE.getSqlTypeName();
+             metaConnection = addColumnsIfNotExists(metaConnection,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, columnsToAdd);
+         }
+         if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) {
+             // Drop old stats table so that new stats table is created
+             metaConnection = dropStatsTable(metaConnection,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 4);
+             metaConnection = addColumnsIfNotExists(
+               metaConnection,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 3,
+               PhoenixDatabaseMetaData.TRANSACTIONAL + " "
+                 + PBoolean.INSTANCE.getSqlTypeName());
+             metaConnection = addColumnsIfNotExists(
+               metaConnection,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 2,
+               PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " "
+                 + PLong.INSTANCE.getSqlTypeName());
+             metaConnection = setImmutableTableIndexesImmutable(metaConnection,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1);
+             metaConnection = updateSystemCatalogTimestamp(metaConnection,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
+             ConnectionQueryServicesImpl.this.removeTable(null,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
+             clearCache();
+         }
+ 
+         if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0) {
+             metaConnection = addColumnsIfNotExists(
+               metaConnection,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 - 2,
+               PhoenixDatabaseMetaData.IS_NAMESPACE_MAPPED + " "
+                 + PBoolean.INSTANCE.getSqlTypeName());
+             metaConnection = addColumnsIfNotExists(
+               metaConnection,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 - 1,
+               PhoenixDatabaseMetaData.AUTO_PARTITION_SEQ + " "
+                 + PVarchar.INSTANCE.getSqlTypeName());
+             metaConnection = addColumnsIfNotExists(
+               metaConnection,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0,
+               PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA + " "
+                 + PBoolean.INSTANCE.getSqlTypeName());
+             metaConnection = UpgradeUtil.disableViewIndexes(metaConnection);
+             if (getProps().getBoolean(QueryServices.LOCAL_INDEX_CLIENT_UPGRADE_ATTRIB,
+               QueryServicesOptions.DEFAULT_LOCAL_INDEX_CLIENT_UPGRADE)) {
+                 metaConnection = UpgradeUtil.upgradeLocalIndexes(metaConnection);
+             }
+             ConnectionQueryServicesImpl.this.removeTable(null,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0);
+             clearCache();
+         }
+         if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0) {
+             metaConnection = addColumnsIfNotExists(
+               metaConnection,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0,
+               PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + " "
+                 + PLong.INSTANCE.getSqlTypeName());
+             ConnectionQueryServicesImpl.this.removeTable(null,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0);
+             clearCache();
+         }
+         if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0) {
+             metaConnection = addColumnQualifierColumn(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 3);
+             metaConnection = addColumnsIfNotExists(
+               metaConnection,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 2,
+               PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME + " "
+                 + PTinyint.INSTANCE.getSqlTypeName());
+             metaConnection = addColumnsIfNotExists(
+               metaConnection,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 1,
+               PhoenixDatabaseMetaData.ENCODING_SCHEME + " "
+                 + PTinyint.INSTANCE.getSqlTypeName());
+             metaConnection = addColumnsIfNotExists(
+               metaConnection,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0,
+               PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER + " "
+                 + PInteger.INSTANCE.getSqlTypeName());
+             ConnectionQueryServicesImpl.this.removeTable(null,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0);
+             clearCache();
+         }
+         if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0) {
+             metaConnection = addColumnsIfNotExists(
+               metaConnection,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0,
+               PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION + " "
+                 + PBoolean.INSTANCE.getSqlTypeName());
+             addParentToChildLinks(metaConnection);
+         }
+         if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0) {
+             metaConnection = addColumnsIfNotExists(
+               metaConnection,
+               PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+               MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0,
+               PhoenixDatabaseMetaData.TRANSACTION_PROVIDER + " "
+                 + PTinyint.INSTANCE.getSqlTypeName());
+         }
++        // TODO set the version for which the following upgrade code runs correct
++        if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0) {
++            addViewIndexToParentLinks(metaConnection);
++            moveChildLinks(metaConnection);
++        }
+         return metaConnection;
+     }
+ 
+     /**
       * There is no other locking needed here since only one connection (on the same or different JVM) will be able to
       * acquire the upgrade mutex via {@link #acquireUpgradeMutex(long, byte[])}.
       */
@@@ -2957,11 -3113,17 +3123,20 @@@
                  }
              }
              try {
-                 metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_FUNCTION_METADATA);
+                 metaConnection.createStatement().executeUpdate(getFunctionTableDDL());
+             } catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {}
+             try {
+                 metaConnection.createStatement().executeUpdate(getLogTableDDL());
              } catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {}
 +            try {
 +                metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_CHILD_LINK_METADATA);
 +            } catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {}
+ 
+             // In case namespace mapping is enabled and system table to system namespace mapping is also enabled,
+             // create an entry for the SYSTEM namespace in the SYSCAT table, so that GRANT/REVOKE commands can work
+             // with SYSTEM Namespace
+             createSchemaIfNotExistsSystemNSMappingEnabled(metaConnection);
+ 
              ConnectionQueryServicesImpl.this.upgradeRequired.set(false);
              success = true;
          } catch (UpgradeInProgressException | UpgradeNotRequiredException e) {


[05/50] [abbrv] phoenix git commit: PHOENIX-4672 Align the kerberos principal config keys

Posted by td...@apache.org.
PHOENIX-4672 Align the kerberos principal config keys

Fix the configuration keys in a manner that won't break users
who are using the 'old' name.


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/2e6dc818
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/2e6dc818
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/2e6dc818

Branch: refs/heads/system-catalog
Commit: 2e6dc818d74b9b0aade96ca9f85f30abe8b9e42e
Parents: ed63661
Author: Josh Elser <el...@apache.org>
Authored: Tue Apr 10 14:08:57 2018 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Apr 10 14:28:51 2018 -0400

----------------------------------------------------------------------
 .../src/main/java/org/apache/phoenix/query/QueryServices.java    | 3 ++-
 .../java/org/apache/phoenix/queryserver/server/QueryServer.java  | 4 ++++
 2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/2e6dc818/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 43b9e5a..21f043c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -237,7 +237,8 @@ public interface QueryServices extends SQLCloseable {
     public static final String QUERY_SERVER_KEYTAB_FILENAME_ATTRIB = "phoenix.queryserver.keytab.file";
     public static final String QUERY_SERVER_HTTP_KEYTAB_FILENAME_ATTRIB = "phoenix.queryserver.http.keytab.file";
     public static final String QUERY_SERVER_KERBEROS_PRINCIPAL_ATTRIB = "phoenix.queryserver.kerberos.principal";
-    public static final String QUERY_SERVER_KERBEROS_HTTP_PRINCIPAL_ATTRIB = "phoenix.queryserver.kerberos.http.principal";
+    public static final String QUERY_SERVER_KERBEROS_HTTP_PRINCIPAL_ATTRIB_LEGACY = "phoenix.queryserver.kerberos.http.principal";
+    public static final String QUERY_SERVER_KERBEROS_HTTP_PRINCIPAL_ATTRIB = "phoenix.queryserver.http.kerberos.principal";
     public static final String QUERY_SERVER_DNS_NAMESERVER_ATTRIB = "phoenix.queryserver.dns.nameserver";
     public static final String QUERY_SERVER_DNS_INTERFACE_ATTRIB = "phoenix.queryserver.dns.interface";
     public static final String QUERY_SERVER_HBASE_SECURITY_CONF_ATTRIB = "hbase.security.authentication";

http://git-wip-us.apache.org/repos/asf/phoenix/blob/2e6dc818/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/QueryServer.java
----------------------------------------------------------------------
diff --git a/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/QueryServer.java b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/QueryServer.java
index 6b1fcfe..8436086 100644
--- a/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/QueryServer.java
+++ b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/QueryServer.java
@@ -246,6 +246,10 @@ public final class QueryServer extends Configured implements Tool, Runnable {
         File keytab = new File(keytabPath);
         String httpKeytabPath = getConf().get(QueryServices.QUERY_SERVER_HTTP_KEYTAB_FILENAME_ATTRIB, null);
         String httpPrincipal = getConf().get(QueryServices.QUERY_SERVER_KERBEROS_HTTP_PRINCIPAL_ATTRIB, null);
+        // Backwards compat for a configuration key change
+        if (httpPrincipal == null) {
+          httpPrincipal = getConf().get(QueryServices.QUERY_SERVER_KERBEROS_HTTP_PRINCIPAL_ATTRIB_LEGACY, null);
+        }
         File httpKeytab = null;
         if (null != httpKeytabPath)
           httpKeytab = new File(httpKeytabPath);


[08/50] [abbrv] phoenix git commit: PHOENIX-4658 IllegalStateException: requestSeek cannot be called on ReversedKeyValueHeap (Toshihiro Suzuki)

Posted by td...@apache.org.
PHOENIX-4658 IllegalStateException: requestSeek cannot be called on ReversedKeyValueHeap (Toshihiro Suzuki)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/1a312a9c
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/1a312a9c
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/1a312a9c

Branch: refs/heads/system-catalog
Commit: 1a312a9c9769ce40485405a386a48686b985b35f
Parents: e40d920
Author: James Taylor <jt...@salesforce.com>
Authored: Wed Apr 11 13:31:38 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Apr 11 13:40:22 2018 -0700

----------------------------------------------------------------------
 .../phoenix/end2end/MultiCfQueryExecIT.java     | 47 ++++++++++++++++++++
 .../apache/phoenix/compile/OrderByCompiler.java | 11 +++--
 .../apache/phoenix/compile/QueryCompiler.java   | 12 +----
 .../java/org/apache/phoenix/parse/HintNode.java |  4 ++
 .../java/org/apache/phoenix/util/ScanUtil.java  |  2 +
 5 files changed, 62 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/1a312a9c/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java
index d94df6c..01da2d8 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java
@@ -31,11 +31,13 @@ import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.sql.Statement;
 import java.util.List;
 import java.util.Properties;
 
 import org.apache.phoenix.query.KeyRange;
 import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
 import org.apache.phoenix.util.SchemaUtil;
 import org.apache.phoenix.util.TestUtil;
 import org.junit.Before;
@@ -406,4 +408,49 @@ public class MultiCfQueryExecIT extends ParallelStatsEnabledIT {
             assertFalse(rs.next());
         }
     }
+
+    @Test
+    public void testBug4658() throws Exception {
+        try (Connection conn = DriverManager.getConnection(getUrl());
+          Statement stmt = conn.createStatement()) {
+            String tableName = generateUniqueName();
+
+            stmt.execute("CREATE TABLE " + tableName + " ("
+                + "COL1 VARCHAR NOT NULL,"
+                + "COL2 VARCHAR NOT NULL,"
+                + "COL3 VARCHAR,"
+                + "FAM.COL4 VARCHAR,"
+                + "CONSTRAINT TRADE_EVENT_PK PRIMARY KEY (COL1, COL2))");
+            stmt.execute("UPSERT INTO " + tableName + " (COL1, COL2) values ('111', 'AAA')");
+            stmt.execute("UPSERT INTO " + tableName + " (COL1, COL2) values ('222', 'AAA')");
+            conn.commit();
+
+            try (ResultSet rs = stmt.executeQuery(
+              "SELECT * FROM " + tableName + " WHERE COL2 = 'AAA' ORDER BY COL1 DESC")) {
+                assertTrue(rs.next());
+                assertEquals(rs.getString("COL1"), "222");
+                assertEquals(rs.getString("COL2"), "AAA");
+                assertTrue(rs.next());
+                assertEquals(rs.getString("COL1"), "111");
+                assertEquals(rs.getString("COL2"), "AAA");
+                assertFalse(rs.next());
+            }
+
+            // Tests for FORWARD_SCAN hint
+            String query = "SELECT /*+ FORWARD_SCAN */ * FROM " + tableName + " WHERE COL2 = 'AAA' ORDER BY COL1 DESC";
+            try (ResultSet rs = stmt.executeQuery("EXPLAIN " + query)) {
+                String explainPlan = QueryUtil.getExplainPlan(rs);
+                assertFalse(explainPlan.contains("REVERSE"));
+            }
+            try (ResultSet rs = stmt.executeQuery(query)) {
+                assertTrue(rs.next());
+                assertEquals(rs.getString("COL1"), "222");
+                assertEquals(rs.getString("COL2"), "AAA");
+                assertTrue(rs.next());
+                assertEquals(rs.getString("COL1"), "111");
+                assertEquals(rs.getString("COL2"), "AAA");
+                assertFalse(rs.next());
+            }
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/1a312a9c/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderByCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderByCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderByCompiler.java
index 1097f70..b83c7a8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderByCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderByCompiler.java
@@ -30,6 +30,7 @@ import org.apache.phoenix.exception.SQLExceptionInfo;
 import org.apache.phoenix.execute.TupleProjector;
 import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.expression.OrderByExpression;
+import org.apache.phoenix.parse.HintNode.Hint;
 import org.apache.phoenix.parse.LiteralParseNode;
 import org.apache.phoenix.parse.OrderByNode;
 import org.apache.phoenix.parse.ParseNode;
@@ -154,12 +155,16 @@ public class OrderByCompiler {
         // If we're ordering by the order returned by the scan, we don't need an order by
         if (isInRowKeyOrder && tracker.isOrderPreserving()) {
             if (tracker.isReverse()) {
-                // Don't use reverse scan if we're using a skip scan, as our skip scan doesn't support this yet.
+                // Don't use reverse scan if:
+                // 1) we're using a skip scan, as our skip scan doesn't support this yet.
+                // 2) we have the FORWARD_SCAN hint set to choose to keep loading of column
+                //    families on demand versus doing a reverse scan
                 // REV_ROW_KEY_ORDER_BY scan would not take effect for a projected table, so don't return it for such table types.
                 if (context.getConnection().getQueryServices().getProps().getBoolean(QueryServices.USE_REVERSE_SCAN_ATTRIB, QueryServicesOptions.DEFAULT_USE_REVERSE_SCAN)
                         && !context.getScanRanges().useSkipScanFilter()
                         && context.getCurrentTable().getTable().getType() != PTableType.PROJECTED
-                        && context.getCurrentTable().getTable().getType() != PTableType.SUBQUERY) {
+                        && context.getCurrentTable().getTable().getType() != PTableType.SUBQUERY
+                        && !statement.getHint().hasHint(Hint.FORWARD_SCAN)) {
                     return OrderBy.REV_ROW_KEY_ORDER_BY;
                 }
             } else {
@@ -172,4 +177,4 @@ public class OrderByCompiler {
 
     private OrderByCompiler() {
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/1a312a9c/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
index 9568ad8..3e5f5ee 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
@@ -66,7 +66,6 @@ import org.apache.phoenix.parse.SelectStatement;
 import org.apache.phoenix.parse.SubqueryParseNode;
 import org.apache.phoenix.parse.TableNode;
 import org.apache.phoenix.query.ConnectionQueryServices;
-import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.query.QueryServicesOptions;
 import org.apache.phoenix.schema.AmbiguousColumnException;
@@ -91,13 +90,6 @@ import com.google.common.collect.Sets;
  */
 public class QueryCompiler {
     private static final ParseNodeFactory NODE_FACTORY = new ParseNodeFactory();
-    /*
-     * Not using Scan.setLoadColumnFamiliesOnDemand(true) because we don't
-     * want to introduce a dependency on 0.94.5 (where this feature was
-     * introduced). This will do the same thing. Once we do have a
-     * dependency on 0.94.5 or above, switch this around.
-     */
-    private static final String LOAD_COLUMN_FAMILIES_ON_DEMAND_ATTR = "_ondemand_";
     private final PhoenixStatement statement;
     private final Scan scan;
     private final Scan originalScan;
@@ -128,9 +120,7 @@ public class QueryCompiler {
         this.noChildParentJoinOptimization = select.getHint().hasHint(Hint.NO_CHILD_PARENT_JOIN_OPTIMIZATION);
         ConnectionQueryServices services = statement.getConnection().getQueryServices();
         this.costBased = services.getProps().getBoolean(QueryServices.COST_BASED_OPTIMIZER_ENABLED, QueryServicesOptions.DEFAULT_COST_BASED_OPTIMIZER_ENABLED);
-        if (services.getLowestClusterHBaseVersion() >= PhoenixDatabaseMetaData.ESSENTIAL_FAMILY_VERSION_THRESHOLD) {
-            this.scan.setAttribute(LOAD_COLUMN_FAMILIES_ON_DEMAND_ATTR, QueryConstants.TRUE);
-        }
+        scan.setLoadColumnFamiliesOnDemand(true);
         if (select.getHint().hasHint(Hint.NO_CACHE)) {
             scan.setCacheBlocks(false);
         }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/1a312a9c/phoenix-core/src/main/java/org/apache/phoenix/parse/HintNode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/HintNode.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/HintNode.java
index 6d8451b..39e9b05 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/parse/HintNode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/HintNode.java
@@ -104,6 +104,10 @@ public class HintNode {
       * Enforces a serial scan.
       */
      SERIAL,
+        /**
+         * Enforces a forward scan.
+         */
+        FORWARD_SCAN,
     };
 
     private final Map<Hint,String> hints;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/1a312a9c/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
index 9c710c1..dd885fd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -612,10 +612,12 @@ public class ScanUtil {
     
     public static void setReversed(Scan scan) {
         scan.setAttribute(BaseScannerRegionObserver.REVERSE_SCAN, PDataType.TRUE_BYTES);
+        scan.setLoadColumnFamiliesOnDemand(false);
     }
 
     public static void unsetReversed(Scan scan) {
         scan.setAttribute(BaseScannerRegionObserver.REVERSE_SCAN, PDataType.FALSE_BYTES);
+        scan.setLoadColumnFamiliesOnDemand(true);
     }
 
     private static byte[] getReversedRow(byte[] startRow) {


[37/50] [abbrv] phoenix git commit: PHOENIX-4698 Tolerate orphaned views (Maddineni Sukumar)

Posted by td...@apache.org.
PHOENIX-4698 Tolerate orphaned views (Maddineni Sukumar)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8dd637dc
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8dd637dc
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8dd637dc

Branch: refs/heads/system-catalog
Commit: 8dd637dc41ae9fa1cb9c3111481e216059f95c34
Parents: 041e7b5
Author: James Taylor <jt...@salesforce.com>
Authored: Thu Apr 19 14:42:24 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Thu Apr 26 09:55:28 2018 -0700

----------------------------------------------------------------------
 .../coprocessor/MetaDataEndpointImpl.java       | 50 +++++++++++++-------
 1 file changed, 33 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/8dd637dc/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index c28ad3c..ddd3ffe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -592,7 +592,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
             done.run(builder.build());
             return;
         } catch (Throwable t) {
-        	logger.error("getTable failed", t);
+            logger.error("getTable failed", t);
             ProtobufUtil.setControllerException(controller,
                 ServerUtil.createIOException(SchemaUtil.getTableName(schemaName, tableName), t));
         }
@@ -755,9 +755,9 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                                                                                                // compatibility.
         Cell sortOrderKv = colKeyValues[SORT_ORDER_INDEX];
         SortOrder sortOrder =
-        		sortOrderKv == null ? SortOrder.getDefault() : SortOrder.fromSystemValue(PInteger.INSTANCE
+                sortOrderKv == null ? SortOrder.getDefault() : SortOrder.fromSystemValue(PInteger.INSTANCE
                         .getCodec().decodeInt(sortOrderKv.getValueArray(),
-                        		sortOrderKv.getValueOffset(), SortOrder.getDefault()));
+                                sortOrderKv.getValueOffset(), SortOrder.getDefault()));
 
         Cell arraySizeKv = colKeyValues[ARRAY_SIZE_INDEX];
         Integer arraySize = arraySizeKv == null ? null :
@@ -1313,9 +1313,9 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
         return table.getName() == null;
     }
 
-	private static boolean isSchemaDeleted(PSchema schema) {
-		return schema.getSchemaName() == null;
-	}
+    private static boolean isSchemaDeleted(PSchema schema) {
+        return schema.getSchemaName() == null;
+    }
 
     private static boolean isFunctionDeleted(PFunction function) {
         return function.getFunctionName() == null;
@@ -1827,6 +1827,13 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
             byte[] tableKey = SchemaUtil.getTableKey(viewtenantId, viewSchema, viewTable);
             ImmutableBytesPtr cacheKey = new ImmutableBytesPtr(tableKey);
             PTable view = loadTable(env, tableKey, cacheKey, clientTimeStamp, clientTimeStamp, clientVersion);
+            if (view == null) {
+                logger.warn("Found orphan tenant view row in SYSTEM.CATALOG with tenantId:"
+                        + Bytes.toString(tenantId) + ", schema:"
+                        + Bytes.toString(viewSchema) + ", table:"
+                        + Bytes.toString(viewTable));
+                continue;
+            }
             findAllChildViews(region, viewtenantId, view, result, clientTimeStamp, clientVersion);
         }
     }
@@ -1969,8 +1976,9 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
         if (systemCatalog.getTimeStamp() < MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0) {
             return findChildViews_deprecated(region, tenantId, table, PHYSICAL_TABLE_BYTES, stopAfterFirst);
         } else {
-            return findChildViews_4_11(region, tenantId, table.getSchemaName().getBytes(),
-                table.getTableName().getBytes(), stopAfterFirst);
+            return findChildViews_4_11(region, tenantId, 
+                    table.getSchemaName() == null ? ByteUtil.EMPTY_BYTE_ARRAY : table.getSchemaName().getBytes(),
+                    table.getTableName().getBytes(), stopAfterFirst);
         }
     }
     
@@ -2548,6 +2556,14 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
             // lock the rows corresponding to views so that no other thread can modify the view meta-data
             RowLock viewRowLock = acquireLock(region, viewKey, locks);
             PTable view = doGetTable(viewKey, clientTimeStamp, viewRowLock, clientVersion);
+            if (view == null) {
+                logger.warn("Found orphan tenant view row in SYSTEM.CATALOG with tenantId:"
+                    + Bytes.toString(tenantId) + ", schema:"
+                    + Bytes.toString(schema) + ", table:"
+                    + Bytes.toString(table));
+                continue;
+             }
+            
             ColumnOrdinalPositionUpdateList ordinalPositionList = new ColumnOrdinalPositionUpdateList();
             List<PColumn> viewPkCols = new ArrayList<>(view.getPKColumns());
             boolean addingExistingPkCol = false;
@@ -2705,12 +2721,12 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                 for (TableProperty tableProp : TableProperty.values()) {
                     Cell tablePropertyCell = tablePropertyCellMap.get(tableProp);
                     if ( tablePropertyCell != null) {
-						// set this table property on the view :
-						// 1. if it is not mutable on a view (which means the property is always the same as the base table)
-						// 2. or if it is mutable on a view and if it doesn't exist on the view
-						// 3. or if it is mutable on a view and the property value is the same as the base table property (which means it wasn't changed on the view)
+                        // set this table property on the view :
+                        // 1. if it is not mutable on a view (which means the property is always the same as the base table)
+                        // 2. or if it is mutable on a view and if it doesn't exist on the view
+                        // 3. or if it is mutable on a view and the property value is the same as the base table property (which means it wasn't changed on the view)
                         Object viewProp = tableProp.getPTableValue(view);
-						if (!tableProp.isMutableOnView() || viewProp==null || viewProp.equals(tableProp.getPTableValue(basePhysicalTable))) {
+                        if (!tableProp.isMutableOnView() || viewProp==null || viewProp.equals(tableProp.getPTableValue(basePhysicalTable))) {
                             viewHeaderRowPut.add(CellUtil.createCell(viewKey, CellUtil.cloneFamily(tablePropertyCell),
                                 CellUtil.cloneQualifier(tablePropertyCell), clientTimeStamp, tablePropertyCell.getTypeByte(),
                                 CellUtil.cloneValue(tablePropertyCell)));
@@ -2800,10 +2816,10 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
         
         // if switching from from non tx to tx
         if (!basePhysicalTable.isTransactional() && switchAttribute(basePhysicalTable, basePhysicalTable.isTransactional(), tableMetadata, TRANSACTIONAL_BYTES)) {
-        	invalidateList.add(new ImmutableBytesPtr(viewKey));
-        	Put put = new Put(viewKey);
-            put.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
-            		TRANSACTIONAL_BYTES, clientTimeStamp, PBoolean.INSTANCE.toBytes(true));
+            invalidateList.add(new ImmutableBytesPtr(viewKey));
+            Put put = new Put(viewKey);
+            put.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                    TRANSACTIONAL_BYTES, clientTimeStamp, PBoolean.INSTANCE.toBytes(true));
             mutationsForAddingColumnsToViews.add(put);
         }
     }


[12/50] [abbrv] phoenix git commit: PHOENIX-4605 Support running multiple transaction providers

Posted by td...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index a7b31e8..1a11427 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -69,6 +69,7 @@ import org.apache.phoenix.schema.types.PDataType;
 import org.apache.phoenix.schema.types.PDouble;
 import org.apache.phoenix.schema.types.PFloat;
 import org.apache.phoenix.schema.types.PVarchar;
+import org.apache.phoenix.transaction.TransactionFactory;
 import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.PhoenixRuntime;
@@ -134,7 +135,7 @@ public class PTableImpl implements PTable {
     private boolean disableWAL;
     private boolean multiTenant;
     private boolean storeNulls;
-    private boolean isTransactional;
+    private TransactionFactory.Provider transactionProvider;
     private ViewType viewType;
     private Short viewIndexId;
     private int estimatedSize;
@@ -227,7 +228,7 @@ public class PTableImpl implements PTable {
         init(tenantId, this.schemaName, this.tableName, PTableType.INDEX, state, timeStamp, sequenceNumber, pkName, bucketNum, columns,
             this.schemaName, parentTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName,
             null, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable,
-            isTransactional, updateCacheFrequency, indexDisableTimestamp, isNamespaceMpped, null, false, storageScheme, qualifierEncodingScheme, encodedCQCounter, useStatsForParallelization);
+            transactionProvider, updateCacheFrequency, indexDisableTimestamp, isNamespaceMpped, null, false, storageScheme, qualifierEncodingScheme, encodedCQCounter, useStatsForParallelization);
     }
 
     public PTableImpl(long timeStamp) { // For delete marker
@@ -270,7 +271,7 @@ public class PTableImpl implements PTable {
                     table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), getColumnsToClone(table), table.getParentSchemaName(), table.getParentTableName(),
                     indexes, table.isImmutableRows(), physicalNames, table.getDefaultFamilyName(), viewStatement,
                     table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
-                    table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), updateCacheFrequency,
+                    table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.getTransactionProvider(), updateCacheFrequency,
                     table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
         }
 
@@ -280,7 +281,7 @@ public class PTableImpl implements PTable {
                 table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), getColumnsToClone(table), parentSchemaName, table.getParentTableName(),
                 indexes, table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), viewStatement,
                 table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
-                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(),
+                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.getTransactionProvider(), table.getUpdateCacheFrequency(),
                 table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
     }
 
@@ -290,7 +291,7 @@ public class PTableImpl implements PTable {
                 table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(),
                 table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
                 table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
-                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(),
+                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.getTransactionProvider(), table.getUpdateCacheFrequency(),
                 table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
     }
     
@@ -300,7 +301,7 @@ public class PTableImpl implements PTable {
                 table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(),
                 table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
                 table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
-                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(),
+                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.getTransactionProvider(), table.getUpdateCacheFrequency(),
                 table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
     }
 
@@ -310,7 +311,7 @@ public class PTableImpl implements PTable {
                 table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(),
                 table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), defaultFamily, table.getViewStatement(),
                 table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
-                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(),
+                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.getTransactionProvider(), table.getUpdateCacheFrequency(),
                 table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
     }
 
@@ -320,7 +321,7 @@ public class PTableImpl implements PTable {
                 sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(),
                 table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(),
                 table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
-                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), 
+                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.getTransactionProvider(), table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), 
                 table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
     }
 
@@ -330,18 +331,18 @@ public class PTableImpl implements PTable {
                 sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(),
                 table.getIndexes(), isImmutableRows, table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
                 table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(),
-                table.getIndexType(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(),
+                table.getIndexType(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.getTransactionProvider(),
                 table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
     }
     
     public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, Collection<PColumn> columns, boolean isImmutableRows, boolean isWalDisabled,
-            boolean isMultitenant, boolean storeNulls, boolean isTransactional, long updateCacheFrequency, boolean isNamespaceMapped) throws SQLException {
+            boolean isMultitenant, boolean storeNulls, TransactionFactory.Provider transactionProvider, long updateCacheFrequency, boolean isNamespaceMapped) throws SQLException {
         return new PTableImpl(
                 table.getTenantId(), table.getSchemaName(), table.getTableName(), table.getType(), table.getIndexState(), timeStamp,
                 sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(),
                 table.getIndexes(), isImmutableRows, table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
                 isWalDisabled, isMultitenant, storeNulls, table.getViewType(), table.getViewIndexId(), table.getIndexType(),
-                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), isTransactional, updateCacheFrequency, table.getIndexDisableTimestamp(), 
+                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), transactionProvider, updateCacheFrequency, table.getIndexDisableTimestamp(), 
                 isNamespaceMapped, table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
     }
     
@@ -352,7 +353,7 @@ public class PTableImpl implements PTable {
                 table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(),
                 table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
                 table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
-                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(),
+                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.getTransactionProvider(), table.getUpdateCacheFrequency(),
                 table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
     }
 
@@ -363,7 +364,7 @@ public class PTableImpl implements PTable {
                 table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(),
                 table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
                 table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
-                table.getBaseColumnCount(), rowKeyOrderOptimizable, table.isTransactional(), table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), table.isNamespaceMapped(), 
+                table.getBaseColumnCount(), rowKeyOrderOptimizable, table.getTransactionProvider(), table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), table.isNamespaceMapped(), 
                 table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
     }
 
@@ -374,7 +375,7 @@ public class PTableImpl implements PTable {
                 table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(),
                 table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
                 table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
-                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), 
+                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.getTransactionProvider(), table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), 
                 table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
     }
 
@@ -383,12 +384,12 @@ public class PTableImpl implements PTable {
             Collection<PColumn> columns, PName dataSchemaName, PName dataTableName, List<PTable> indexes,
             boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression,
             boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId,
-            IndexType indexType, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency,
+            IndexType indexType, boolean rowKeyOrderOptimizable, TransactionFactory.Provider transactionProvider, long updateCacheFrequency,
             long indexDisableTimestamp, boolean isNamespaceMapped, String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme, QualifierEncodingScheme qualifierEncodingScheme, EncodedCQCounter encodedCQCounter, Boolean useStatsForParallelization) throws SQLException {
         return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, dataSchemaName,
                 dataTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName,
                 viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId,
-                indexType, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, rowKeyOrderOptimizable, isTransactional,
+                indexType, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, rowKeyOrderOptimizable, transactionProvider,
                 updateCacheFrequency,indexDisableTimestamp, isNamespaceMapped, autoPartitionSeqName, isAppendOnlySchema, storageScheme, qualifierEncodingScheme, encodedCQCounter, useStatsForParallelization);
     }
 
@@ -397,7 +398,7 @@ public class PTableImpl implements PTable {
             Collection<PColumn> columns, PName dataSchemaName, PName dataTableName, List<PTable> indexes,
             boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression,
             boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId,
-            IndexType indexType, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency,
+            IndexType indexType, boolean rowKeyOrderOptimizable, TransactionFactory.Provider transactionProvider, long updateCacheFrequency,
             int baseColumnCount, long indexDisableTimestamp, boolean isNamespaceMapped,
             String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme,
             QualifierEncodingScheme qualifierEncodingScheme, EncodedCQCounter encodedCQCounter, Boolean useStatsForParallelization)
@@ -405,7 +406,7 @@ public class PTableImpl implements PTable {
         return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName,
                 bucketNum, columns, dataSchemaName, dataTableName, indexes, isImmutableRows, physicalNames,
                 defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId,
-                indexType, baseColumnCount, rowKeyOrderOptimizable, isTransactional, updateCacheFrequency, 
+                indexType, baseColumnCount, rowKeyOrderOptimizable, transactionProvider, updateCacheFrequency, 
                 indexDisableTimestamp, isNamespaceMapped, autoPartitionSeqName, isAppendOnlySchema, storageScheme, qualifierEncodingScheme, encodedCQCounter, useStatsForParallelization);
     }
 
@@ -414,13 +415,13 @@ public class PTableImpl implements PTable {
             PName parentSchemaName, PName parentTableName, List<PTable> indexes, boolean isImmutableRows,
             List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant,
             boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType,
-            int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency,
+            int baseColumnCount, boolean rowKeyOrderOptimizable, TransactionFactory.Provider transactionProvider, long updateCacheFrequency,
             long indexDisableTimestamp, boolean isNamespaceMapped, String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme, 
             QualifierEncodingScheme qualifierEncodingScheme, EncodedCQCounter encodedCQCounter, Boolean useStatsForParallelization) throws SQLException {
         init(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns,
                 parentSchemaName, parentTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName,
                 viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable,
-                isTransactional, updateCacheFrequency, indexDisableTimestamp, isNamespaceMapped, autoPartitionSeqName, isAppendOnlySchema, storageScheme, 
+                transactionProvider, updateCacheFrequency, indexDisableTimestamp, isNamespaceMapped, autoPartitionSeqName, isAppendOnlySchema, storageScheme, 
                 qualifierEncodingScheme, encodedCQCounter, useStatsForParallelization);
     }
     
@@ -454,7 +455,7 @@ public class PTableImpl implements PTable {
             PName pkName, Integer bucketNum, Collection<PColumn> columns, PName parentSchemaName, PName parentTableName,
             List<PTable> indexes, boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL,
             boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId,
-            IndexType indexType , int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency, long indexDisableTimestamp, 
+            IndexType indexType , int baseColumnCount, boolean rowKeyOrderOptimizable, TransactionFactory.Provider transactionProvider, long updateCacheFrequency, long indexDisableTimestamp, 
             boolean isNamespaceMapped, String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme, QualifierEncodingScheme qualifierEncodingScheme, 
             EncodedCQCounter encodedCQCounter, Boolean useStatsForParallelization) throws SQLException {
         Preconditions.checkNotNull(schemaName);
@@ -486,7 +487,7 @@ public class PTableImpl implements PTable {
         this.viewType = viewType;
         this.viewIndexId = viewIndexId;
         this.indexType = indexType;
-        this.isTransactional = isTransactional;
+        this.transactionProvider = transactionProvider;
         this.rowKeyOrderOptimizable = rowKeyOrderOptimizable;
         this.updateCacheFrequency = updateCacheFrequency;
         this.isNamespaceMapped = isNamespaceMapped;
@@ -1279,7 +1280,13 @@ public class PTableImpl implements PTable {
         boolean disableWAL = table.getDisableWAL();
         boolean multiTenant = table.getMultiTenant();
         boolean storeNulls = table.getStoreNulls();
-        boolean isTransactional = table.getTransactional();
+        TransactionFactory.Provider transactionProvider = null;
+        if (table.hasTransactionProvider()) {
+            transactionProvider = TransactionFactory.Provider.fromCode(table.getTransactionProvider());
+        } else if (table.hasTransactional()) {
+            // For backward compatibility prior to transactionProvider field
+            transactionProvider = TransactionFactory.Provider.TEPHRA;
+        }
         ViewType viewType = null;
         String viewStatement = null;
         List<PName> physicalNames = Collections.emptyList();
@@ -1352,7 +1359,7 @@ public class PTableImpl implements PTable {
                 (bucketNum == NO_SALTING) ? null : bucketNum, columns, parentSchemaName, parentTableName, indexes,
                         isImmutableRows, physicalNames, defaultFamilyName, viewStatement, disableWAL,
                         multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable,
-                        isTransactional, updateCacheFrequency, indexDisableTimestamp, isNamespaceMapped, autoParititonSeqName, 
+                        transactionProvider, updateCacheFrequency, indexDisableTimestamp, isNamespaceMapped, autoParititonSeqName, 
                         isAppendOnlySchema, storageScheme, qualifierEncodingScheme, encodedColumnQualifierCounter, useStatsForParallelization);
             return result;
         } catch (SQLException e) {
@@ -1418,7 +1425,9 @@ public class PTableImpl implements PTable {
       builder.setDisableWAL(table.isWALDisabled());
       builder.setMultiTenant(table.isMultiTenant());
       builder.setStoreNulls(table.getStoreNulls());
-      builder.setTransactional(table.isTransactional());
+      if (table.getTransactionProvider() != null) {
+          builder.setTransactionProvider(table.getTransactionProvider().getCode());
+      }
       if(table.getType() == PTableType.VIEW){
         builder.setViewType(ByteStringer.wrap(new byte[]{table.getViewType().getSerializedValue()}));
       }
@@ -1473,8 +1482,13 @@ public class PTableImpl implements PTable {
     }
 
     @Override
-    public boolean isTransactional() {
-        return isTransactional;
+    public TransactionFactory.Provider getTransactionProvider() {
+        return transactionProvider;
+    }
+    
+    @Override
+    public final boolean isTransactional() {
+        return transactionProvider != null;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
index c500b2e..78b9beb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
@@ -33,6 +33,7 @@ import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.exception.SQLExceptionInfo;
 import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
+import org.apache.phoenix.transaction.TransactionFactory;
 import org.apache.phoenix.util.SchemaUtil;
 
 public enum TableProperty {
@@ -94,6 +95,23 @@ public enum TableProperty {
         }
     },
     
+    TRANSACTION_PROVIDER(PhoenixDatabaseMetaData.TRANSACTION_PROVIDER, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, false, false) {
+        @Override
+        public Object getPTableValue(PTable table) {
+            return table.getTransactionProvider();
+        }
+        @Override
+        public Object getValue(Object value) {
+            try {
+                return value == null ? null : TransactionFactory.Provider.valueOf(value.toString());
+            } catch (IllegalArgumentException e) {
+                throw new RuntimeException(new SQLExceptionInfo.Builder(SQLExceptionCode.UNKNOWN_TRANSACTION_PROVIDER)
+                .setMessage(value.toString())
+                .build().buildException());
+            }
+        }
+    },
+
     UPDATE_CACHE_FREQUENCY(PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY, true, true, true) {
 	    @Override
         public Object getValue(Object value) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionContext.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionContext.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionContext.java
index 110868e..543eda1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionContext.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionContext.java
@@ -17,17 +17,11 @@
  */
 package org.apache.phoenix.transaction;
 
-import java.io.IOException;
 import java.sql.SQLException;
-import java.util.concurrent.TimeoutException;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
-import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
+import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.phoenix.schema.PTable;
-import org.apache.phoenix.util.ReadOnlyProps;
-import org.apache.twill.zookeeper.ZKClientService;
-import org.slf4j.Logger;
+import org.apache.phoenix.transaction.TransactionFactory.Provider;
 
 public class OmidTransactionContext implements PhoenixTransactionContext {
 
@@ -56,7 +50,7 @@ public class OmidTransactionContext implements PhoenixTransactionContext {
     }
 
     @Override
-    public void commitDDLFence(PTable dataTable, Logger logger) throws SQLException {
+    public void commitDDLFence(PTable dataTable) throws SQLException {
         // TODO Auto-generated method stub
 
     }
@@ -116,59 +110,24 @@ public class OmidTransactionContext implements PhoenixTransactionContext {
     }
 
     @Override
-    public long getMaxTransactionsPerSecond() {
-        // TODO Auto-generated method stub
-        return 0;
-    }
-
-    @Override
-    public boolean isPreExistingVersion(long version) {
-        // TODO Auto-generated method stub
-        return false;
+    public Provider getProvider() {
+        return Provider.OMID;
     }
 
     @Override
-    public BaseRegionObserver getCoprocessor() {
-        // TODO Auto-generated method stub
+    public PhoenixTransactionContext newTransactionContext(PhoenixTransactionContext contex, boolean subTask) {
         return null;
     }
 
     @Override
-    public void setInMemoryTransactionClient(Configuration config) {
+    public void markDMLFence(PTable dataTable) {
         // TODO Auto-generated method stub
         
     }
 
     @Override
-    public ZKClientService setTransactionClient(Configuration config, ReadOnlyProps props,
-            ConnectionInfo connectionInfo) {
-        // TODO Auto-generated method stub
-        
-        return null;
-        
-    }
-
-    @Override
-    public byte[] getFamilyDeleteMarker() {
+    public HTableInterface getTransactionalTable(HTableInterface htable, boolean isImmutable) {
         // TODO Auto-generated method stub
         return null;
     }
-
-    @Override
-    public void setTxnConfigs(Configuration config, String tmpFolder, int defaultTxnTimeoutSeconds) throws IOException {
-        // TODO Auto-generated method stub
-
-    }
-
-    @Override
-    public void setupTxManager(Configuration config, String url) throws SQLException {
-        // TODO Auto-generated method stub
-
-    }
-
-    @Override
-    public void tearDownTxManager() {
-        // TODO Auto-generated method stub
-
-    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionProvider.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionProvider.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionProvider.java
index b0c1bfe..c211661 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionProvider.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionProvider.java
@@ -19,34 +19,28 @@ package org.apache.phoenix.transaction;
 
 import java.io.IOException;
 
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
 import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
+import org.apache.phoenix.transaction.TransactionFactory.Provider;
 
-public class OmidTransactionProvider implements TransactionProvider {
+public class OmidTransactionProvider implements PhoenixTransactionProvider {
     private static final OmidTransactionProvider INSTANCE = new OmidTransactionProvider();
-    
+
     public static final OmidTransactionProvider getInstance() {
         return INSTANCE;
     }
-    
+
     private OmidTransactionProvider() {
     }
-    
-    @Override
-    public PhoenixTransactionContext getTransactionContext()  {
-        return new OmidTransactionContext();
-    }
 
     @Override
     public PhoenixTransactionContext getTransactionContext(byte[] txnBytes) throws IOException {
         //return new OmidTransactionContext(txnBytes);
         return null;
     }
-    
+
     @Override
     public PhoenixTransactionContext getTransactionContext(PhoenixConnection connection) {
         //return new OmidTransactionContext(connection);
@@ -54,25 +48,37 @@ public class OmidTransactionProvider implements TransactionProvider {
     }
 
     @Override
-    public PhoenixTransactionContext getTransactionContext(PhoenixTransactionContext contex, PhoenixConnection connection, boolean subTask) {
-        //return new OmidTransactionContext(contex, connection, subTask);
+    public PhoenixTransactionClient getTransactionClient(Configuration config, ConnectionInfo connectionInfo) {
+        // TODO Auto-generated method stub
         return null;
     }
 
     @Override
-    public PhoenixTransactionalTable getTransactionalTable(PhoenixTransactionContext ctx, HTableInterface htable) {
-        //return new OmidTransactionTable(ctx, htable);
+    public PhoenixTransactionService getTransactionService(Configuration config, ConnectionInfo connectionInfo) {
+        // TODO Auto-generated method stub
         return null;
     }
-    
+
     @Override
-    public Cell newDeleteFamilyMarker(byte[] row, byte[] family, long timestamp) {
-        return CellUtil.createCell(row, family, HConstants.EMPTY_BYTE_ARRAY, timestamp, KeyValue.Type.Put.getCode(), HConstants.EMPTY_BYTE_ARRAY);
+    public Class<? extends RegionObserver> getCoprocessor() {
+        // TODO Auto-generated method stub
+        return null;
     }
-    
+
     @Override
-    public Cell newDeleteColumnMarker(byte[] row, byte[] family, byte[] qualifier, long timestamp) {
-        return CellUtil.createCell(row, family, qualifier, timestamp, KeyValue.Type.Put.getCode(), HConstants.EMPTY_BYTE_ARRAY);
+    public Provider getProvider() {
+        return TransactionFactory.Provider.OMID;
     }
 
+    @Override
+    public boolean isUnsupported(Feature feature) {
+        // FIXME: if we initialize a Set with the unsupported features
+        // and check for containment, we run into a test failure
+        // in SetPropertyOnEncodedTableIT.testSpecifyingColumnFamilyForTTLFails()
+        // due to TableProperty.colFamSpecifiedException being null
+        // (though it's set in the constructor). I suspect some
+        // mysterious class loader issue. The below works fine
+        // as a workaround.
+        return (feature == Feature.ALTER_NONTX_TO_TX);
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java
deleted file mode 100644
index afbf6ab..0000000
--- a/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java
+++ /dev/null
@@ -1,388 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.transaction;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Append;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Increment;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Row;
-import org.apache.hadoop.hbase.client.RowMutations;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.coprocessor.Batch.Call;
-import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
-
-import com.google.protobuf.Descriptors.MethodDescriptor;
-import com.google.protobuf.Message;
-import com.google.protobuf.Service;
-import com.google.protobuf.ServiceException;
-
-public class OmidTransactionTable implements PhoenixTransactionalTable {
-
-    public OmidTransactionTable(PhoenixTransactionContext ctx, HTableInterface hTable) {
-        // TODO Auto-generated constructor stub
-    }
-
-    @Override
-    public Result get(Get get) throws IOException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public void put(Put put) throws IOException {
-        // TODO Auto-generated method stub
-
-    }
-
-    @Override
-    public void delete(Delete delete) throws IOException {
-        // TODO Auto-generated method stub
-
-    }
-
-    @Override
-    public ResultScanner getScanner(Scan scan) throws IOException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public byte[] getTableName() {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Configuration getConfiguration() {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public HTableDescriptor getTableDescriptor() throws IOException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public boolean exists(Get get) throws IOException {
-        // TODO Auto-generated method stub
-        return false;
-    }
-
-    @Override
-    public Result[] get(List<Get> gets) throws IOException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public ResultScanner getScanner(byte[] family) throws IOException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public ResultScanner getScanner(byte[] family, byte[] qualifier)
-            throws IOException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public void put(List<Put> puts) throws IOException {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public void delete(List<Delete> deletes) throws IOException {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public void setAutoFlush(boolean autoFlush) {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public boolean isAutoFlush() {
-        // TODO Auto-generated method stub
-        return false;
-    }
-
-    @Override
-    public long getWriteBufferSize() {
-        // TODO Auto-generated method stub
-        return 0;
-    }
-
-    @Override
-    public void setWriteBufferSize(long writeBufferSize) throws IOException {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public void flushCommits() throws IOException {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public void close() throws IOException {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public long incrementColumnValue(byte[] row, byte[] family,
-            byte[] qualifier, long amount, boolean writeToWAL)
-            throws IOException {
-        // TODO Auto-generated method stub
-        return 0;
-    }
-
-    @Override
-    public Boolean[] exists(List<Get> gets) throws IOException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public void setAutoFlushTo(boolean autoFlush) {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public TableName getName() {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public boolean[] existsAll(List<Get> gets) throws IOException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public void batch(List<? extends Row> actions, Object[] results)
-            throws IOException, InterruptedException {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public Object[] batch(List<? extends Row> actions) throws IOException,
-            InterruptedException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public <R> void batchCallback(List<? extends Row> actions,
-            Object[] results, Callback<R> callback) throws IOException,
-            InterruptedException {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public <R> Object[] batchCallback(List<? extends Row> actions,
-            Callback<R> callback) throws IOException, InterruptedException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier,
-            byte[] value, Put put) throws IOException {
-        // TODO Auto-generated method stub
-        return false;
-    }
-
-    @Override
-    public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier,
-            CompareOp compareOp, byte[] value, Put put) throws IOException {
-        // TODO Auto-generated method stub
-        return false;
-    }
-
-    @Override
-    public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
-            byte[] value, Delete delete) throws IOException {
-        // TODO Auto-generated method stub
-        return false;
-    }
-
-    @Override
-    public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
-            CompareOp compareOp, byte[] value, Delete delete)
-            throws IOException {
-        // TODO Auto-generated method stub
-        return false;
-    }
-
-    @Override
-    public void mutateRow(RowMutations rm) throws IOException {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public Result append(Append append) throws IOException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Result increment(Increment increment) throws IOException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public long incrementColumnValue(byte[] row, byte[] family,
-            byte[] qualifier, long amount) throws IOException {
-        // TODO Auto-generated method stub
-        return 0;
-    }
-
-    @Override
-    public long incrementColumnValue(byte[] row, byte[] family,
-            byte[] qualifier, long amount, Durability durability)
-            throws IOException {
-        // TODO Auto-generated method stub
-        return 0;
-    }
-
-    @Override
-    public CoprocessorRpcChannel coprocessorService(byte[] row) {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public <T extends Service, R> Map<byte[], R> coprocessorService(
-            Class<T> service, byte[] startKey, byte[] endKey,
-            Call<T, R> callable) throws ServiceException, Throwable {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public <T extends Service, R> void coprocessorService(Class<T> service,
-            byte[] startKey, byte[] endKey, Call<T, R> callable,
-            Callback<R> callback) throws ServiceException, Throwable {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public <R extends Message> Map<byte[], R> batchCoprocessorService(
-            MethodDescriptor methodDescriptor, Message request,
-            byte[] startKey, byte[] endKey, R responsePrototype)
-            throws ServiceException, Throwable {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public <R extends Message> void batchCoprocessorService(
-            MethodDescriptor methodDescriptor, Message request,
-            byte[] startKey, byte[] endKey, R responsePrototype,
-            Callback<R> callback) throws ServiceException, Throwable {
-        // TODO Auto-generated method stub
-    }
-
-    @Override
-    public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier,
-            CompareOp compareOp, byte[] value, RowMutations mutation)
-            throws IOException {
-        // TODO Auto-generated method stub
-        return false;
-    }
-
-    @Override
-    public int getOperationTimeout() {
-        // TODO Auto-generated method stub
-        return 0;
-    }
-
-    @Override
-    public int getRpcTimeout() {
-        // TODO Auto-generated method stub
-        return 0;
-    }
-
-    @Override
-    public void setOperationTimeout(int arg0) {
-        // TODO Auto-generated method stub
-        
-    }
-
-    @Override
-    public void setRpcTimeout(int arg0) {
-        // TODO Auto-generated method stub
-        
-    }
-
-    @Override
-    public int getReadRpcTimeout() {
-      // TODO Auto-generated method stub
-      return 0;
-    }
-
-    @Override
-    public void setReadRpcTimeout(int readRpcTimeout) {
-      // TODO Auto-generated method stub
-      
-    }
-
-    @Override
-    public int getWriteRpcTimeout() {
-      // TODO Auto-generated method stub
-      return 0;
-    }
-
-    @Override
-    public void setWriteRpcTimeout(int writeRpcTimeout) {
-      // TODO Auto-generated method stub
-      
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionClient.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionClient.java
new file mode 100644
index 0000000..f12f818
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionClient.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.transaction;
+
+import java.io.Closeable;
+
+public interface PhoenixTransactionClient extends Closeable {
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionContext.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionContext.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionContext.java
index 52ff2f9..f3ad42f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionContext.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionContext.java
@@ -17,20 +17,98 @@
  */
 package org.apache.phoenix.transaction;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
-import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
-import org.apache.phoenix.schema.PTable;
-import org.apache.phoenix.util.ReadOnlyProps;
-import org.apache.twill.zookeeper.ZKClientService;
-import org.slf4j.Logger;
-
-import java.io.IOException;
 import java.sql.SQLException;
 import java.util.concurrent.TimeoutException;
 
-public interface PhoenixTransactionContext {
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.transaction.TransactionFactory.Provider;
 
+public interface PhoenixTransactionContext {
+    public static PhoenixTransactionContext NULL_CONTEXT = new PhoenixTransactionContext() {
+
+        @Override
+        public void begin() throws SQLException {
+        }
+
+        @Override
+        public void commit() throws SQLException {
+        }
+
+        @Override
+        public void abort() throws SQLException {
+        }
+
+        @Override
+        public void checkpoint(boolean hasUncommittedData) throws SQLException {
+        }
+
+        @Override
+        public void commitDDLFence(PTable dataTable) throws SQLException {
+        }
+
+        @Override
+        public void join(PhoenixTransactionContext ctx) {
+        }
+
+        @Override
+        public boolean isTransactionRunning() {
+            return false;
+        }
+
+        @Override
+        public void reset() {
+        }
+
+        @Override
+        public long getTransactionId() {
+            return 0;
+        }
+
+        @Override
+        public long getReadPointer() {
+            return 0;
+        }
+
+        @Override
+        public long getWritePointer() {
+            return 0;
+        }
+
+        @Override
+        public void setVisibilityLevel(PhoenixVisibilityLevel visibilityLevel) {
+        }
+
+        @Override
+        public PhoenixVisibilityLevel getVisibilityLevel() {
+            return null;
+        }
+
+        @Override
+        public byte[] encodeTransaction() throws SQLException {
+            return null;
+        }
+
+        @Override
+        public Provider getProvider() {
+            return null;
+        }
+
+        @Override
+        public PhoenixTransactionContext newTransactionContext(PhoenixTransactionContext contex, boolean subTask) {
+            return NULL_CONTEXT;
+        }
+
+        @Override
+        public void markDMLFence(PTable dataTable) {
+            
+        }
+
+        @Override
+        public HTableInterface getTransactionalTable(HTableInterface htable, boolean isImmutable) {
+            return null;
+        }
+    };
     /**
      * 
      * Visibility levels needed for checkpointing and  
@@ -49,22 +127,6 @@ public interface PhoenixTransactionContext {
     public static final String READ_NON_TX_DATA = "data.tx.read.pre.existing";
 
     /**
-     * Set the in memory client connection to the transaction manager (for testing purpose)
-     *
-     * @param config
-     */
-    public void setInMemoryTransactionClient(Configuration config);
-
-    /**
-     * Set the client connection to the transaction manager
-     *
-     * @param config
-     * @param props
-     * @param connectionInfo
-     */
-    public ZKClientService setTransactionClient(Configuration config, ReadOnlyProps props, ConnectionInfo connectionInfo);
-
-    /**
      * Starts a transaction
      *
      * @throws SQLException
@@ -84,7 +146,7 @@ public interface PhoenixTransactionContext {
      * @throws SQLException
      */
     public void abort() throws SQLException;
-
+    
     /**
      * Create a checkpoint in a transaction as defined in [TEPHRA-96]
      * @throws SQLException
@@ -100,9 +162,17 @@ public interface PhoenixTransactionContext {
      * @throws InterruptedException
      * @throws TimeoutException
      */
-    public void commitDDLFence(PTable dataTable, Logger logger)
+    public void commitDDLFence(PTable dataTable)
             throws SQLException;
 
+
+    /**
+     * Mark the start of DML go ensure that updates to indexed rows are not
+     * missed.
+     * @param dataTable the table on which DML command is working
+     */
+    public void markDMLFence(PTable dataTable);
+
     /**
      * Augment the current context with ctx modified keys
      *
@@ -121,7 +191,8 @@ public interface PhoenixTransactionContext {
     public void reset();
 
     /**
-     * Returns transaction unique identifier
+     * Returns transaction unique identifier which is also
+     * assumed to be the earliest write pointer.
      */
     public long getTransactionId();
 
@@ -150,42 +221,8 @@ public interface PhoenixTransactionContext {
      */
     public byte[] encodeTransaction() throws SQLException;
 
-    /**
-     * 
-     * @return max transactions per second
-     */
-    public long getMaxTransactionsPerSecond();
+    public Provider getProvider();
+    public PhoenixTransactionContext newTransactionContext(PhoenixTransactionContext contex, boolean subTask);
 
-    /**
-     *
-     * @param version
-     */
-    public boolean isPreExistingVersion(long version);
-
-    /**
-     *
-     * @return the coprocessor
-     */
-    public BaseRegionObserver getCoprocessor();
-
-    /**
-     * 
-     * @return the family delete marker
-     */
-    public byte[] getFamilyDeleteMarker();
-
-    /**
-     * Setup transaction manager's configuration for testing
-     */
-     public void setTxnConfigs(Configuration config, String tmpFolder, int defaultTxnTimeoutSeconds) throws IOException;
-
-    /**
-     * Setup transaction manager for testing
-     */
-    public void setupTxManager(Configuration config, String url) throws SQLException;
-
-    /**
-     * Tear down transaction manager for testing
-     */
-    public void tearDownTxManager();
+    public HTableInterface getTransactionalTable(HTableInterface htable, boolean isImmutable);
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionProvider.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionProvider.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionProvider.java
new file mode 100644
index 0000000..cdc6058
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionProvider.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.transaction;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
+
+public interface PhoenixTransactionProvider {
+    public enum Feature {
+        ALTER_NONTX_TO_TX(SQLExceptionCode.CANNOT_ALTER_TABLE_FROM_NON_TXN_TO_TXNL);
+        
+        private final SQLExceptionCode code;
+        
+        Feature(SQLExceptionCode code) {
+            this.code = code;
+        }
+        
+        public SQLExceptionCode getCode() {
+            return code;
+        }
+    }
+    public PhoenixTransactionContext getTransactionContext(byte[] txnBytes) throws IOException;
+    public PhoenixTransactionContext getTransactionContext(PhoenixConnection connection);
+    
+    public PhoenixTransactionClient getTransactionClient(Configuration config, ConnectionInfo connectionInfo);
+    public PhoenixTransactionService getTransactionService(Configuration config, ConnectionInfo connectionInfo);
+    public Class<? extends RegionObserver> getCoprocessor();
+    
+    public TransactionFactory.Provider getProvider();
+    public boolean isUnsupported(Feature feature);
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionService.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionService.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionService.java
new file mode 100644
index 0000000..10c46e1
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionService.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.transaction;
+
+import java.io.Closeable;
+
+public interface PhoenixTransactionService extends Closeable {
+
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionalTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionalTable.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionalTable.java
deleted file mode 100644
index 7af1c08..0000000
--- a/phoenix-core/src/main/java/org/apache/phoenix/transaction/PhoenixTransactionalTable.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.transaction;
-
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-
-import java.io.IOException;
-import java.util.List;
-
-public interface PhoenixTransactionalTable extends HTableInterface {
-
-    /**
-     * Transaction version of {@link HTableInterface#get(Get get)}
-     * @param get
-     * @throws IOException
-     */
-    public Result get(Get get) throws IOException;
-
-    /**
-     * Transactional version of {@link HTableInterface#put(Put put)}
-     * @param put
-     * @throws IOException
-     */
-    public void put(Put put) throws IOException;
-
-    /**
-     * Transactional version of {@link HTableInterface#delete(Delete delete)}
-     *
-     * @param delete
-     * @throws IOException
-     */
-    public void delete(Delete delete) throws IOException;
-
-    /**
-     * Transactional version of {@link HTableInterface#getScanner(Scan scan)}
-     *
-     * @param scan
-     * @return ResultScanner
-     * @throws IOException
-     */
-    public ResultScanner getScanner(Scan scan) throws IOException;
-
-    /**
-     * Returns Htable name
-     */
-    public byte[] getTableName();
-
-    /**
-     * Returns Htable configuration object
-     */
-    public Configuration getConfiguration();
-
-    /**
-     * Returns HTableDescriptor of Htable
-     * @throws IOException
-     */
-    public HTableDescriptor getTableDescriptor() throws IOException;
-
-    /**
-     * Checks if cell exists
-     * @throws IOException
-     */
-    public boolean exists(Get get) throws IOException;
-
-    /**
-     * Transactional version of {@link HTableInterface#get(List gets)}
-     * @throws IOException
-     */
-    public Result[] get(List<Get> gets) throws IOException;
-
-    /**
-     * Transactional version of {@link HTableInterface#getScanner(byte[] family)}
-     * @throws IOException
-     */
-    public ResultScanner getScanner(byte[] family) throws IOException;
-
-    /**
-     * Transactional version of {@link HTableInterface#getScanner(byte[] family, byte[] qualifier)}
-     * @throws IOException
-     */
-    public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException;
-
-    /**
-     * Transactional version of {@link HTableInterface#put(List puts)}
-     * @throws IOException
-     */
-    public void put(List<Put> puts) throws IOException;
-
-    /**
-     * Transactional version of {@link HTableInterface#delete(List deletes)}
-     * @throws IOException
-     */
-    public void delete(List<Delete> deletes) throws IOException;
-
-    /**
-     * Delegates to {@link HTable#setAutoFlush(boolean autoFlush)}
-     */
-    public void setAutoFlush(boolean autoFlush);
-
-    /**
-     * Delegates to {@link HTable#isAutoFlush()}
-     */
-    public boolean isAutoFlush();
-
-    /**
-     * Delegates to see HTable.getWriteBufferSize()
-     */
-    public long getWriteBufferSize();
-
-    /**
-     * Delegates to see HTable.setWriteBufferSize()
-     */
-    public void setWriteBufferSize(long writeBufferSize) throws IOException;
-
-    /**
-     * Delegates to see HTable.flushCommits()
-     */
-    public void flushCommits() throws IOException;
-
-    /**
-     * Releases resources
-     * @throws IOException
-     */
-    public void close() throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionContext.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionContext.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionContext.java
index 77c3ab6..8b16210 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionContext.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionContext.java
@@ -19,71 +19,46 @@ package org.apache.phoenix.transaction;
 
 import java.io.IOException;
 import java.sql.SQLException;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
 import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.exception.SQLExceptionInfo;
 import org.apache.phoenix.jdbc.PhoenixConnection;
-import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
 import org.apache.phoenix.schema.PTable;
-import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.transaction.TephraTransactionProvider.TephraTransactionClient;
 import org.apache.tephra.Transaction;
+import org.apache.tephra.Transaction.VisibilityLevel;
 import org.apache.tephra.TransactionAware;
 import org.apache.tephra.TransactionCodec;
 import org.apache.tephra.TransactionConflictException;
 import org.apache.tephra.TransactionContext;
 import org.apache.tephra.TransactionFailureException;
-import org.apache.tephra.TransactionManager;
 import org.apache.tephra.TransactionSystemClient;
-import org.apache.tephra.Transaction.VisibilityLevel;
 import org.apache.tephra.TxConstants;
-import org.apache.tephra.distributed.PooledClientProvider;
-import org.apache.tephra.distributed.TransactionServiceClient;
-import org.apache.tephra.hbase.coprocessor.TransactionProcessor;
-import org.apache.tephra.inmemory.InMemoryTxSystemClient;
-import org.apache.tephra.util.TxUtils;
+import org.apache.tephra.hbase.TransactionAwareHTable;
 import org.apache.tephra.visibility.FenceWait;
 import org.apache.tephra.visibility.VisibilityFence;
-import org.apache.tephra.zookeeper.TephraZKClientService;
-import org.apache.tephra.distributed.TransactionService;
-import org.apache.tephra.metrics.TxMetricsCollector;
-import org.apache.tephra.persist.HDFSTransactionStateStorage;
-import org.apache.tephra.snapshot.SnapshotCodecProvider;
-import org.apache.twill.discovery.DiscoveryService;
-import org.apache.twill.discovery.ZKDiscoveryService;
-import org.apache.twill.internal.utils.Networks;
-import org.apache.twill.zookeeper.RetryStrategies;
-import org.apache.twill.zookeeper.ZKClientService;
-import org.apache.twill.zookeeper.ZKClientServices;
-import org.apache.twill.zookeeper.ZKClients;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Lists;
-import com.google.inject.util.Providers;
-
 import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-public class TephraTransactionContext implements PhoenixTransactionContext {
+import com.google.common.collect.Lists;
 
+public class TephraTransactionContext implements PhoenixTransactionContext {
+    private static final Logger logger = LoggerFactory.getLogger(TephraTransactionContext.class);
     private static final TransactionCodec CODEC = new TransactionCodec();
 
-    private static TransactionSystemClient txClient = null;
-    private static ZKClientService zkClient = null;
-    private static TransactionService txService = null;
-    private static TransactionManager txManager = null;
-
     private final List<TransactionAware> txAwares;
     private final TransactionContext txContext;
     private Transaction tx;
     private TransactionSystemClient txServiceClient;
-    private TransactionFailureException e;
 
     public TephraTransactionContext() {
         this.txServiceClient = null;
@@ -93,21 +68,21 @@ public class TephraTransactionContext implements PhoenixTransactionContext {
 
     public TephraTransactionContext(byte[] txnBytes) throws IOException {
         this();
-        this.tx = (txnBytes != null && txnBytes.length > 0) ? CODEC
-                .decode(txnBytes) : null;
+        this.tx = CODEC.decode(txnBytes);
     }
 
     public TephraTransactionContext(PhoenixConnection connection) {
-        this.txServiceClient = txClient;  
+        PhoenixTransactionClient client = connection.getQueryServices().initTransactionClient(getProvider());  
+        assert (client instanceof TephraTransactionClient);
+        this.txServiceClient = ((TephraTransactionClient)client).getTransactionClient();
         this.txAwares = Collections.emptyList();
         this.txContext = new TransactionContext(txServiceClient);
     }
 
-    public TephraTransactionContext(PhoenixTransactionContext ctx,
-            PhoenixConnection connection, boolean subTask) {
-        this.txServiceClient = txClient;  
+    private TephraTransactionContext(PhoenixTransactionContext ctx, boolean subTask) {
         assert (ctx instanceof TephraTransactionContext);
         TephraTransactionContext tephraTransactionContext = (TephraTransactionContext) ctx;
+        this.txServiceClient = tephraTransactionContext.txServiceClient;  
 
         if (subTask) {
             this.tx = tephraTransactionContext.getTransaction();
@@ -117,42 +92,13 @@ public class TephraTransactionContext implements PhoenixTransactionContext {
             this.txAwares = Collections.emptyList();
             this.txContext = tephraTransactionContext.getContext();
         }
-
-        this.e = null;
-    }
-
-    @Override
-    public void setInMemoryTransactionClient(Configuration config) {
-        TransactionManager txnManager = new TransactionManager(config);
-        txClient = this.txServiceClient = new InMemoryTxSystemClient(txnManager);
     }
 
     @Override
-    public ZKClientService setTransactionClient(Configuration config, ReadOnlyProps props, ConnectionInfo connectionInfo) {
-        String zkQuorumServersString = props.get(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM);
-        if (zkQuorumServersString==null) {
-            zkQuorumServersString = connectionInfo.getZookeeperQuorum()+":"+connectionInfo.getPort();
-        }
-
-        int timeOut = props.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
-        // Create instance of the tephra zookeeper client
-        ZKClientService txZKClientService  = ZKClientServices.delegate(
-            ZKClients.reWatchOnExpire(
-                ZKClients.retryOnFailure(
-                     new TephraZKClientService(zkQuorumServersString, timeOut, null,
-                             ArrayListMultimap.<String, byte[]>create()), 
-                         RetryStrategies.exponentialDelay(500, 2000, TimeUnit.MILLISECONDS))
-                     )
-                );
-        txZKClientService.startAndWait();
-        ZKDiscoveryService zkDiscoveryService = new ZKDiscoveryService(txZKClientService);
-        PooledClientProvider pooledClientProvider = new PooledClientProvider(
-                config, zkDiscoveryService);
-        txClient = this.txServiceClient = new TransactionServiceClient(config,pooledClientProvider);
-        
-        return txZKClientService;
+    public TransactionFactory.Provider getProvider() {
+        return TransactionFactory.Provider.TEPHRA;
     }
-
+    
     @Override
     public void begin() throws SQLException {
         if (txContext == null) {
@@ -181,8 +127,6 @@ public class TephraTransactionContext implements PhoenixTransactionContext {
         try {
             txContext.finish();
         } catch (TransactionFailureException e) {
-            this.e = e;
-
             if (e instanceof TransactionConflictException) {
                 throw new SQLExceptionInfo.Builder(
                         SQLExceptionCode.TRANSACTION_CONFLICT_EXCEPTION)
@@ -204,14 +148,8 @@ public class TephraTransactionContext implements PhoenixTransactionContext {
         }
 
         try {
-            if (e != null) {
-                txContext.abort(e);
-                e = null;
-            } else {
-                txContext.abort();
-            }
+            txContext.abort();
         } catch (TransactionFailureException e) {
-            this.e = null;
             throw new SQLExceptionInfo.Builder(
                     SQLExceptionCode.TRANSACTION_FAILED)
                     .setMessage(e.getMessage()).setRootCause(e).build()
@@ -249,7 +187,7 @@ public class TephraTransactionContext implements PhoenixTransactionContext {
     }
 
     @Override
-    public void commitDDLFence(PTable dataTable, Logger logger)
+    public void commitDDLFence(PTable dataTable)
             throws SQLException {
         byte[] key = dataTable.getName().getBytes();
 
@@ -276,7 +214,12 @@ public class TephraTransactionContext implements PhoenixTransactionContext {
         }
     }
 
+    @Override
     public void markDMLFence(PTable table) {
+        if (table.getType() == PTableType.INDEX) {
+            return;
+        }
+        
         byte[] logicalKey = table.getName().getBytes();
         TransactionAware logicalTxAware = VisibilityFence.create(logicalKey);
 
@@ -300,6 +243,9 @@ public class TephraTransactionContext implements PhoenixTransactionContext {
 
     @Override
     public void join(PhoenixTransactionContext ctx) {
+        if (ctx == PhoenixTransactionContext.NULL_CONTEXT) {
+            return;
+        }
         assert (ctx instanceof TephraTransactionContext);
         TephraTransactionContext tephraContext = (TephraTransactionContext) ctx;
 
@@ -325,7 +271,6 @@ public class TephraTransactionContext implements PhoenixTransactionContext {
     public void reset() {
         tx = null;
         txAwares.clear();
-        this.e = null;
     }
 
     @Override
@@ -406,86 +351,15 @@ public class TephraTransactionContext implements PhoenixTransactionContext {
         assert (tx != null);
 
         try {
-            return CODEC.encode(tx);
+            byte[] encodedTxBytes = CODEC.encode(tx);
+            encodedTxBytes = Arrays.copyOf(encodedTxBytes, encodedTxBytes.length + 1);
+            encodedTxBytes[encodedTxBytes.length - 1] = getProvider().getCode();
+            return encodedTxBytes;
         } catch (IOException e) {
             throw new SQLException(e);
         }
     }
 
-    @Override
-    public long getMaxTransactionsPerSecond() {
-        return TxConstants.MAX_TX_PER_MS;
-    }
-
-    @Override
-    public boolean isPreExistingVersion(long version) {
-        return TxUtils.isPreExistingVersion(version);
-    }
-
-    @Override
-    public BaseRegionObserver getCoprocessor() {
-        return new TransactionProcessor();
-    }
-
-    @Override
-    public byte[] getFamilyDeleteMarker() {
-        return TxConstants.FAMILY_DELETE_QUALIFIER;
-    }
-
-    @Override
-    public void setTxnConfigs(Configuration config, String tmpFolder, int defaultTxnTimeoutSeconds) throws IOException {
-        config.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
-        config.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times");
-        config.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1);
-        config.setInt(TxConstants.Service.CFG_DATA_TX_BIND_PORT, Networks.getRandomPort());
-        config.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder);
-        config.setInt(TxConstants.Manager.CFG_TX_TIMEOUT, defaultTxnTimeoutSeconds);
-        config.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
-        config.setLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, 5L);
-    }
-
-    @Override
-    public void setupTxManager(Configuration config, String url) throws SQLException {
-
-        if (txService != null) {
-            return;
-        }
-
-        ConnectionInfo connInfo = ConnectionInfo.create(url);
-        zkClient = ZKClientServices.delegate(
-          ZKClients.reWatchOnExpire(
-            ZKClients.retryOnFailure(
-              ZKClientService.Builder.of(connInfo.getZookeeperConnectionString())
-                .setSessionTimeout(config.getInt(HConstants.ZK_SESSION_TIMEOUT,
-                        HConstants.DEFAULT_ZK_SESSION_TIMEOUT))
-                .build(),
-              RetryStrategies.exponentialDelay(500, 2000, TimeUnit.MILLISECONDS)
-            )
-          )
-        );
-        zkClient.startAndWait();
-
-        DiscoveryService discovery = new ZKDiscoveryService(zkClient);
-        txManager = new TransactionManager(config, new HDFSTransactionStateStorage(config, new SnapshotCodecProvider(config), new TxMetricsCollector()), new TxMetricsCollector());
-        txService = new TransactionService(config, zkClient, discovery, Providers.of(txManager));
-        txService.startAndWait();
-    }
-
-    @Override
-    public void tearDownTxManager() {
-        try {
-            if (txService != null) txService.stopAndWait();
-        } finally {
-            try {
-                if (zkClient != null) zkClient.stopAndWait();
-            } finally {
-                txService = null;
-                zkClient = null;
-                txManager = null;
-            }
-        }
-    }
-
     /**
      * TephraTransactionContext specific functions
      */
@@ -511,4 +385,17 @@ public class TephraTransactionContext implements PhoenixTransactionContext {
             txAware.startTx(tx);
         }
     }
+
+    @Override
+    public PhoenixTransactionContext newTransactionContext(PhoenixTransactionContext context, boolean subTask) {
+        return new TephraTransactionContext(context, subTask);
+    }
+    
+    @Override
+    public HTableInterface getTransactionalTable(HTableInterface htable, boolean isImmutable) {
+        TransactionAwareHTable transactionAwareHTable = new TransactionAwareHTable(htable, isImmutable ? TxConstants.ConflictDetection.NONE : TxConstants.ConflictDetection.ROW);
+        this.addTransactionAware(transactionAwareHTable);
+        return transactionAwareHTable;
+    }
+    
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/de83b8d5/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionProvider.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionProvider.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionProvider.java
index 795be9f..2e52efa 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionProvider.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionProvider.java
@@ -18,16 +18,37 @@
 package org.apache.phoenix.transaction;
 
 import java.io.IOException;
+import java.util.concurrent.TimeUnit;
 
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
+import org.apache.phoenix.coprocessor.TephraTransactionalProcessor;
 import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
+import org.apache.phoenix.transaction.TransactionFactory.Provider;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionSystemClient;
 import org.apache.tephra.TxConstants;
+import org.apache.tephra.distributed.PooledClientProvider;
+import org.apache.tephra.distributed.TransactionService;
+import org.apache.tephra.distributed.TransactionServiceClient;
+import org.apache.tephra.inmemory.InMemoryTxSystemClient;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.persist.HDFSTransactionStateStorage;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+import org.apache.tephra.zookeeper.TephraZKClientService;
+import org.apache.twill.discovery.DiscoveryService;
+import org.apache.twill.discovery.ZKDiscoveryService;
+import org.apache.twill.zookeeper.RetryStrategies;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.apache.twill.zookeeper.ZKClientServices;
+import org.apache.twill.zookeeper.ZKClients;
 
-public class TephraTransactionProvider implements TransactionProvider {
+import com.google.common.collect.ArrayListMultimap;
+import com.google.inject.util.Providers;
+
+public class TephraTransactionProvider implements PhoenixTransactionProvider {
     private static final TephraTransactionProvider INSTANCE = new TephraTransactionProvider();
     
     public static final TephraTransactionProvider getInstance() {
@@ -37,12 +58,6 @@ public class TephraTransactionProvider implements TransactionProvider {
     private TephraTransactionProvider() {
     }
     
-    
-    @Override
-    public PhoenixTransactionContext getTransactionContext()  {
-        return new TephraTransactionContext();
-    }
-
     @Override
     public PhoenixTransactionContext getTransactionContext(byte[] txnBytes) throws IOException {
        return new TephraTransactionContext(txnBytes);
@@ -54,23 +69,129 @@ public class TephraTransactionProvider implements TransactionProvider {
     }
 
     @Override
-    public PhoenixTransactionContext getTransactionContext(PhoenixTransactionContext contex, PhoenixConnection connection, boolean subTask) {
-        return new TephraTransactionContext(contex, connection, subTask);
+    public PhoenixTransactionClient getTransactionClient(Configuration config, ConnectionInfo connectionInfo) {
+        if (connectionInfo.isConnectionless()) {
+            TransactionManager txnManager = new TransactionManager(config);
+            TransactionSystemClient txClient = new InMemoryTxSystemClient(txnManager);
+            return new TephraTransactionClient(txClient);
+            
+        }
+        String zkQuorumServersString = config.get(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM);
+        if (zkQuorumServersString==null) {
+            zkQuorumServersString = connectionInfo.getZookeeperConnectionString();
+        }
+
+        int timeOut = config.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
+        // Create instance of the tephra zookeeper client
+        ZKClientService zkClientService  = ZKClientServices.delegate(
+            ZKClients.reWatchOnExpire(
+                ZKClients.retryOnFailure(
+                     new TephraZKClientService(zkQuorumServersString, timeOut, null,
+                             ArrayListMultimap.<String, byte[]>create()), 
+                         RetryStrategies.exponentialDelay(500, 2000, TimeUnit.MILLISECONDS))
+                     )
+                );
+        //txZKClientService.startAndWait();
+        ZKDiscoveryService zkDiscoveryService = new ZKDiscoveryService(zkClientService);
+        PooledClientProvider pooledClientProvider = new PooledClientProvider(
+                config, zkDiscoveryService);
+        TransactionServiceClient txClient = new TransactionServiceClient(config,pooledClientProvider);
+        TephraTransactionClient client = new TephraTransactionClient(zkClientService, txClient);
+        client.start();
+        
+        return client;
     }
 
     @Override
-    public PhoenixTransactionalTable getTransactionalTable(PhoenixTransactionContext ctx, HTableInterface htable) {
-        return new TephraTransactionTable(ctx, htable);
+    public PhoenixTransactionService getTransactionService(Configuration config, ConnectionInfo connInfo) {
+        ZKClientService zkClient = ZKClientServices.delegate(
+          ZKClients.reWatchOnExpire(
+            ZKClients.retryOnFailure(
+              ZKClientService.Builder.of(connInfo.getZookeeperConnectionString())
+                .setSessionTimeout(config.getInt(HConstants.ZK_SESSION_TIMEOUT,
+                        HConstants.DEFAULT_ZK_SESSION_TIMEOUT))
+                .build(),
+              RetryStrategies.exponentialDelay(500, 2000, TimeUnit.MILLISECONDS)
+            )
+          )
+        );
+
+        //zkClient.startAndWait();
+        DiscoveryService discovery = new ZKDiscoveryService(zkClient);
+        TransactionManager txManager = new TransactionManager(config, new HDFSTransactionStateStorage(config, new SnapshotCodecProvider(config), new TxMetricsCollector()), new TxMetricsCollector());
+        TransactionService txService = new TransactionService(config, zkClient, discovery, Providers.of(txManager));
+        TephraTransactionService service = new TephraTransactionService(zkClient, txService);
+        //txService.startAndWait();            
+        service.start();
+        return service;
+    }
+
+    static class TephraTransactionService implements PhoenixTransactionService {
+        private final ZKClientService zkClient;
+        private final TransactionService txService;
+
+        public TephraTransactionService(ZKClientService zkClient, TransactionService txService) {
+            this.zkClient = zkClient;
+            this.txService = txService;
+        }
+        
+        public void start() {
+            zkClient.startAndWait();
+            txService.startAndWait();            
+        }
+        
+        @Override
+        public void close() throws IOException {
+            try {
+                if (txService != null) txService.stopAndWait();
+            } finally {
+                if (zkClient != null) zkClient.stopAndWait();
+            }
+        }
+        
     }
     
-    @Override
-    public Cell newDeleteFamilyMarker(byte[] row, byte[] family, long timestamp) {
-        return CellUtil.createCell(row, family, TxConstants.FAMILY_DELETE_QUALIFIER, timestamp, KeyValue.Type.Put.getCode(), HConstants.EMPTY_BYTE_ARRAY);
+    static class TephraTransactionClient implements PhoenixTransactionClient {
+        private final ZKClientService zkClient;
+        private final TransactionSystemClient txClient;
+
+        public TephraTransactionClient(TransactionSystemClient txClient) {
+            this(null, txClient);
+        }
+        
+        public TephraTransactionClient(ZKClientService zkClient, TransactionSystemClient txClient) {
+            this.zkClient = zkClient;
+            this.txClient = txClient;
+        }
+        
+        public void start() {
+            zkClient.startAndWait();
+        }
+        
+        public TransactionSystemClient getTransactionClient() {
+            return txClient;
+        }
+        
+        @Override
+        public void close() throws IOException {
+            zkClient.stopAndWait();
+        }
+        
     }
     
     @Override
-    public Cell newDeleteColumnMarker(byte[] row, byte[] family, byte[] qualifier, long timestamp) {
-        return CellUtil.createCell(row, family, qualifier, timestamp, KeyValue.Type.Put.getCode(), HConstants.EMPTY_BYTE_ARRAY);
+    public Class<? extends RegionObserver> getCoprocessor() {
+        return TephraTransactionalProcessor.class;
+    }
+
+    @Override
+    public Provider getProvider() {
+        return TransactionFactory.Provider.TEPHRA;
+    }
+
+    @Override
+    public boolean isUnsupported(Feature feature) {
+        return false;
     }
 
 }


[07/50] [abbrv] phoenix git commit: PHOENIX-4653 Upgrading from namespace enabled cluster to latest version failing with UpgradeInProgressException(Rajeshbabu)

Posted by td...@apache.org.
PHOENIX-4653 Upgrading from namespace enabled cluster to latest version failing with UpgradeInProgressException(Rajeshbabu)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/e40d9201
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/e40d9201
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/e40d9201

Branch: refs/heads/system-catalog
Commit: e40d920160ce95e2cbb75642100aa01a1af6fdd9
Parents: d48518b
Author: Rajeshbabu Chintaguntla <ra...@apache.org>
Authored: Wed Apr 11 18:16:45 2018 +0530
Committer: Rajeshbabu Chintaguntla <ra...@apache.org>
Committed: Wed Apr 11 18:16:45 2018 +0530

----------------------------------------------------------------------
 .../apache/phoenix/query/ConnectionQueryServicesImpl.java    | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/e40d9201/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 8c7441a..6627a84 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -2678,13 +2678,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
                 long currentServerSideTableTimeStamp = e.getTable().getTimeStamp();
                 sysCatalogTableName = e.getTable().getPhysicalName().getString();
                 if (currentServerSideTableTimeStamp < MIN_SYSTEM_TABLE_TIMESTAMP) {
-                    // Ensure that the SYSTEM.MUTEX table has been created prior
-                    // to attempting to acquire the upgrade mutex. If namespace
-                    // mapping is enabled, we've already done this earlier in the
-                    // upgrade, so no need for a bunch of wasted RPCs.
-                    if (currentServerSideTableTimeStamp <= MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 &&
-                            !SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM,
-                                    ConnectionQueryServicesImpl.this.getProps())) {
+                    if (currentServerSideTableTimeStamp <= MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0) {
                         try (HBaseAdmin admin = getAdmin()) {
                             createSysMutexTableIfNotExists(admin, this.getProps());
                         }


[44/50] [abbrv] phoenix git commit: PHOENIX-4705 Use XMLInputFactory.newInstance() instead of XMLInputFactory.newFactory()

Posted by td...@apache.org.
PHOENIX-4705 Use XMLInputFactory.newInstance() instead of XMLInputFactory.newFactory()


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/6c1a624f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/6c1a624f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/6c1a624f

Branch: refs/heads/system-catalog
Commit: 6c1a624f351926c6b122c722cf4f9c3418a222ae
Parents: 4b84199
Author: James Taylor <jt...@salesforce.com>
Authored: Thu Apr 26 12:37:26 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Apr 27 11:41:56 2018 -0700

----------------------------------------------------------------------
 .../pherf/configuration/XMLConfigParser.java      |  2 +-
 .../pherf/result/impl/XMLResultHandler.java       | 18 ++++++++++--------
 2 files changed, 11 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/6c1a624f/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java
index f3ec12f..a0ee471 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java
@@ -115,7 +115,7 @@ public class XMLConfigParser {
      */
     // TODO Remove static calls
     public static DataModel readDataModel(Path file) throws JAXBException, XMLStreamException {
-        XMLInputFactory xif = XMLInputFactory.newFactory();
+        XMLInputFactory xif = XMLInputFactory.newInstance();
         xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
         xif.setProperty(XMLInputFactory.SUPPORT_DTD, false);
         JAXBContext jaxbContext = JAXBContext.newInstance(DataModel.class);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/6c1a624f/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/impl/XMLResultHandler.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/impl/XMLResultHandler.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/impl/XMLResultHandler.java
index 05b5a2b..87d6806 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/impl/XMLResultHandler.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/impl/XMLResultHandler.java
@@ -18,8 +18,11 @@
 
 package org.apache.phoenix.pherf.result.impl;
 
-import org.apache.phoenix.pherf.result.*;
-import org.apache.phoenix.pherf.result.file.ResultFileDetails;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.Marshaller;
@@ -28,11 +31,10 @@ import javax.xml.stream.XMLInputFactory;
 import javax.xml.stream.XMLStreamReader;
 import javax.xml.transform.stream.StreamSource;
 
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import org.apache.phoenix.pherf.result.DataModelResult;
+import org.apache.phoenix.pherf.result.Result;
+import org.apache.phoenix.pherf.result.ResultValue;
+import org.apache.phoenix.pherf.result.file.ResultFileDetails;
 
 public class XMLResultHandler extends DefaultResultHandler{
 
@@ -74,7 +76,7 @@ public class XMLResultHandler extends DefaultResultHandler{
     }
 
     List<Result> readFromResultFile(File resultsFile) throws Exception {
-        XMLInputFactory xif = XMLInputFactory.newFactory();
+        XMLInputFactory xif = XMLInputFactory.newInstance();
         xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
         xif.setProperty(XMLInputFactory.SUPPORT_DTD, false);
         JAXBContext jaxbContext = JAXBContext.newInstance(DataModelResult.class);


[40/50] [abbrv] phoenix git commit: PHOENIX-4708 Do not propagate GUIDE_POSTS_WIDTH to children

Posted by td...@apache.org.
PHOENIX-4708 Do not propagate GUIDE_POSTS_WIDTH to children


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ff327348
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ff327348
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ff327348

Branch: refs/heads/system-catalog
Commit: ff3273480165c2832cf1dc1bd114a74b93cf9e72
Parents: bebe66c
Author: James Taylor <jt...@salesforce.com>
Authored: Thu Apr 26 09:40:35 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Thu Apr 26 09:55:28 2018 -0700

----------------------------------------------------------------------
 .../phoenix/end2end/AlterTableWithViewsIT.java  |  22 ++
 .../coprocessor/MetaDataEndpointImpl.java       | 115 +++++---
 .../apache/phoenix/schema/TableProperty.java    | 270 ++++++++++---------
 3 files changed, 248 insertions(+), 159 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/ff327348/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
index 237a8d2..e1b1372 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
@@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.sql.Connection;
+import java.sql.DatabaseMetaData;
 import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
@@ -39,12 +40,14 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.coprocessor.TephraTransactionalProcessor;
 import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PNameFactory;
 import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTableKey;
 import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.util.StringUtil;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -166,6 +169,25 @@ public class AlterTableWithViewsIT extends ParallelStatsDisabledIT {
             assertTrue(viewTable2.isImmutableRows());
             // update cache frequency is not propagated to the view since it was altered on the view
             assertEquals(1, viewTable2.getUpdateCacheFrequency());
+
+            long gpw = 1000000;
+            conn.createStatement().execute("ALTER TABLE " + tableName + " SET GUIDE_POSTS_WIDTH=" + gpw);
+            
+            ResultSet rs;
+            DatabaseMetaData md = conn.getMetaData();
+            rs = md.getTables("", "", StringUtil.escapeLike(tableName), null);
+            assertTrue(rs.next());
+            assertEquals(gpw, rs.getLong(PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH));
+            
+            rs = md.getTables(null, "", StringUtil.escapeLike(viewOfTable1), null);
+            assertTrue(rs.next());
+            rs.getLong(PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH);
+            assertTrue(rs.wasNull());
+
+            rs = md.getTables(null, "", StringUtil.escapeLike(viewOfTable2), null);
+            assertTrue(rs.next());
+            rs.getLong(PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH);
+            assertTrue(rs.wasNull());
         } 
     }
     

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ff327348/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 29eee7e..b77f113 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -91,6 +91,7 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -3163,7 +3164,56 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
         byte[] tableNameBytes = index.getTableName().getBytes();
         return ByteUtil.concat(tenantIdBytes, SEPARATOR_BYTE_ARRAY, schemaNameBytes, SEPARATOR_BYTE_ARRAY, tableNameBytes);
     }
+  
     
+    /**
+     * Determines whether or not we have a change that needs to be propagated from a base table
+     * to it's views. For example, a change to GUIDE_POSTS_WIDTH does not need to be propogated
+     * since it's only set on the physical table.
+     * @param table the table being altered
+     * @param rowKeyMetaData the filled in values for schemaName and tableName
+     * @param tableMetaData the metadata passed over from the client
+     * @return true if changes need to be propagated to the views and false otherwise.
+     */
+    private static boolean hasChangesToPropagate(PTable table, byte[][] rowKeyMetaData, List<Mutation> tableMetaData) {
+        boolean hasChangesToPropagate = true;
+        byte[] schemaName = rowKeyMetaData[SCHEMA_NAME_INDEX];
+        byte[] tableName = rowKeyMetaData[TABLE_NAME_INDEX];
+        for (Mutation m : tableMetaData) {
+            byte[] key = m.getRow();
+            int pkCount = getVarChars(key, rowKeyMetaData);
+            if (pkCount >= COLUMN_NAME_INDEX
+                    && Bytes.compareTo(schemaName, rowKeyMetaData[SCHEMA_NAME_INDEX]) == 0
+                    && Bytes.compareTo(tableName, rowKeyMetaData[TABLE_NAME_INDEX]) == 0) {
+                return true;
+            } else {
+                Collection<List<Cell>>cellLists = m.getFamilyCellMap().values();
+                for (List<Cell> cells : cellLists) {
+                    if (cells != null) {
+                        for (Cell cell : cells) {
+                            byte[] qualifier = CellUtil.cloneQualifier(cell);
+                            String columnName = Bytes.toString(qualifier);
+                            try {
+                                // Often Phoenix table properties aren't valid to be set on a view so thus
+                                // do not need to be propogated. Here we check if the column name corresponds
+                                // to a table property and whether that property is valid to set on a view.
+                                TableProperty tableProp = TableProperty.valueOf(columnName);
+                                if (tableProp.propagateToViews()) {
+                                    return true;
+                                } else {
+                                    hasChangesToPropagate = false;
+                                }
+                            } catch (IllegalArgumentException  e) {
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return hasChangesToPropagate;
+    }
+
+
     @Override
     public void addColumn(RpcController controller, final AddColumnRequest request,
             RpcCallback<MetaDataResponse> done) {
@@ -3188,37 +3238,40 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
                     // Size for worst case - all new columns are PK column
                     List<Mutation> mutationsForAddingColumnsToViews = Lists.newArrayListWithExpectedSize(tableMetaData.size() * ( 1 + table.getIndexes().size()));
                     if (type == PTableType.TABLE || type == PTableType.SYSTEM) {
-                        TableViewFinder childViewsResult = new TableViewFinder();
-                        findAllChildViews(region, tenantId, table, childViewsResult, clientTimeStamp, request.getClientVersion());
-                        if (childViewsResult.hasViews()) {
-                            /* 
-                             * Dis-allow if:
-                             * 1) The meta-data for child view/s spans over
-                             * more than one region (since the changes cannot be made in a transactional fashion)
-                             * 
-                             * 2) The base column count is 0 which means that the metadata hasn't been upgraded yet or
-                             * the upgrade is currently in progress.
-                             * 
-                             * 3) If the request is from a client that is older than 4.5 version of phoenix. 
-                             * Starting from 4.5, metadata requests have the client version included in them. 
-                             * We don't want to allow clients before 4.5 to add a column to the base table if it has views.
-                             * 
-                             * 4) Trying to swtich tenancy of a table that has views
-                             */
-                            if (!childViewsResult.allViewsInSingleRegion() 
-                                    || table.getBaseColumnCount() == 0 
-                                    || !request.hasClientVersion()
-                                    || switchAttribute(table, table.isMultiTenant(), tableMetaData, MULTI_TENANT_BYTES)) {
-                                return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION,
-                                        EnvironmentEdgeManager.currentTimeMillis(), null);
-                            } else {
-                                mutationsForAddingColumnsToViews = new ArrayList<>(childViewsResult.getViewInfoList().size() * tableMetaData.size());
-                                MetaDataMutationResult mutationResult = addColumnsAndTablePropertiesToChildViews(table, tableMetaData, mutationsForAddingColumnsToViews, schemaName, tableName, invalidateList, clientTimeStamp,
-                                        childViewsResult, region, locks, request.getClientVersion());
-                                // return if we were not able to add the column successfully
-                                if (mutationResult!=null)
-                                    return mutationResult;
-                            } 
+                        // If change doesn't need to be propagated, don't bother finding children
+                        if (hasChangesToPropagate(table, rowKeyMetaData, tableMetaData)) {
+                            TableViewFinder childViewsResult = new TableViewFinder();
+                            findAllChildViews(region, tenantId, table, childViewsResult, clientTimeStamp, request.getClientVersion());
+                            if (childViewsResult.hasViews()) {
+                                /* 
+                                 * Dis-allow if:
+                                 * 1) The meta-data for child view/s spans over
+                                 * more than one region (since the changes cannot be made in a transactional fashion)
+                                 * 
+                                 * 2) The base column count is 0 which means that the metadata hasn't been upgraded yet or
+                                 * the upgrade is currently in progress.
+                                 * 
+                                 * 3) If the request is from a client that is older than 4.5 version of phoenix. 
+                                 * Starting from 4.5, metadata requests have the client version included in them. 
+                                 * We don't want to allow clients before 4.5 to add a column to the base table if it has views.
+                                 * 
+                                 * 4) Trying to swtich tenancy of a table that has views
+                                 */
+                                if (!childViewsResult.allViewsInSingleRegion() 
+                                        || table.getBaseColumnCount() == 0 
+                                        || !request.hasClientVersion()
+                                        || switchAttribute(table, table.isMultiTenant(), tableMetaData, MULTI_TENANT_BYTES)) {
+                                    return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION,
+                                            EnvironmentEdgeManager.currentTimeMillis(), null);
+                                } else {
+                                    mutationsForAddingColumnsToViews = new ArrayList<>(childViewsResult.getViewInfoList().size() * tableMetaData.size());
+                                    MetaDataMutationResult mutationResult = addColumnsAndTablePropertiesToChildViews(table, tableMetaData, mutationsForAddingColumnsToViews, schemaName, tableName, invalidateList, clientTimeStamp,
+                                            childViewsResult, region, locks, request.getClientVersion());
+                                    // return if we were not able to add the column successfully
+                                    if (mutationResult!=null)
+                                        return mutationResult;
+                                } 
+                            }
                         }
                     } else if (type == PTableType.VIEW
                             && EncodedColumnsUtil.usesEncodedColumnNames(table)) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ff327348/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
index 78b9beb..3d473c4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
@@ -37,44 +37,44 @@ import org.apache.phoenix.transaction.TransactionFactory;
 import org.apache.phoenix.util.SchemaUtil;
 
 public enum TableProperty {
-    
+
     @Deprecated // use the IMMUTABLE keyword while creating the table
-	IMMUTABLE_ROWS(PhoenixDatabaseMetaData.IMMUTABLE_ROWS, true, true, false) {
+    IMMUTABLE_ROWS(PhoenixDatabaseMetaData.IMMUTABLE_ROWS, true, true, false) {
         @Override
         public Object getPTableValue(PTable table) {
             return table.isImmutableRows();
         }
     },
 
-	MULTI_TENANT(PhoenixDatabaseMetaData.MULTI_TENANT, true, false, false) {
+    MULTI_TENANT(PhoenixDatabaseMetaData.MULTI_TENANT, true, false, false) {
         @Override
         public Object getPTableValue(PTable table) {
             return table.isMultiTenant();
         }
     },
 
-	DISABLE_WAL(PhoenixDatabaseMetaData.DISABLE_WAL, true, false, false) {
+    DISABLE_WAL(PhoenixDatabaseMetaData.DISABLE_WAL, true, false, false) {
         @Override
         public Object getPTableValue(PTable table) {
             return table.isWALDisabled();
         }
     },
 
-	SALT_BUCKETS(PhoenixDatabaseMetaData.SALT_BUCKETS, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, false, SALT_ONLY_ON_CREATE_TABLE, false, false) {
+    SALT_BUCKETS(PhoenixDatabaseMetaData.SALT_BUCKETS, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, false, SALT_ONLY_ON_CREATE_TABLE, false, false) {
         @Override
         public Object getPTableValue(PTable table) {
             return table.getBucketNum();
         }
     },
 
-	DEFAULT_COLUMN_FAMILY(DEFAULT_COLUMN_FAMILY_NAME, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, false, DEFAULT_COLUMN_FAMILY_ONLY_ON_CREATE_TABLE, false, false) {
+    DEFAULT_COLUMN_FAMILY(DEFAULT_COLUMN_FAMILY_NAME, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, false, DEFAULT_COLUMN_FAMILY_ONLY_ON_CREATE_TABLE, false, false) {
         @Override
         public Object getPTableValue(PTable table) {
             return table.getDefaultFamilyName();
         }
     },
 
-	TTL(HColumnDescriptor.TTL, COLUMN_FAMILY_NOT_ALLOWED_FOR_TTL, true, CANNOT_ALTER_PROPERTY, false, false) {
+    TTL(HColumnDescriptor.TTL, COLUMN_FAMILY_NOT_ALLOWED_FOR_TTL, true, CANNOT_ALTER_PROPERTY, false, false) {
         @Override
         public Object getPTableValue(PTable table) {
             return null;
@@ -87,14 +87,14 @@ public enum TableProperty {
             return table.getStoreNulls();
         }
     },
-    
+
     TRANSACTIONAL(PhoenixDatabaseMetaData.TRANSACTIONAL, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, false, false) {
         @Override
         public Object getPTableValue(PTable table) {
             return table.isTransactional();
         }
     },
-    
+
     TRANSACTION_PROVIDER(PhoenixDatabaseMetaData.TRANSACTION_PROVIDER, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, false, false) {
         @Override
         public Object getPTableValue(PTable table) {
@@ -113,28 +113,28 @@ public enum TableProperty {
     },
 
     UPDATE_CACHE_FREQUENCY(PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY, true, true, true) {
-	    @Override
+        @Override
         public Object getValue(Object value) {
-	        if (value instanceof String) {
-	            String strValue = (String) value;
-	            if ("ALWAYS".equalsIgnoreCase(strValue)) {
-	                return 0L;
-	            } else if ("NEVER".equalsIgnoreCase(strValue)) {
-	                return Long.MAX_VALUE;
-	            }
-	        } else {
-	            return value == null ? null : ((Number) value).longValue();
-	        }
-	        return value;
-	    }
+            if (value instanceof String) {
+                String strValue = (String) value;
+                if ("ALWAYS".equalsIgnoreCase(strValue)) {
+                    return 0L;
+                } else if ("NEVER".equalsIgnoreCase(strValue)) {
+                    return Long.MAX_VALUE;
+                }
+            } else {
+                return value == null ? null : ((Number) value).longValue();
+            }
+            return value;
+        }
 
         @Override
         public Object getPTableValue(PTable table) {
             return table.getUpdateCacheFrequency();
-        }	    
-	},
-	
-	AUTO_PARTITION_SEQ(PhoenixDatabaseMetaData.AUTO_PARTITION_SEQ, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, false, false, false) {
+        }
+    },
+
+    AUTO_PARTITION_SEQ(PhoenixDatabaseMetaData.AUTO_PARTITION_SEQ, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, false, false, false) {
         @Override
         public Object getValue(Object value) {
             return value == null ? null : SchemaUtil.normalizeIdentifier(value.toString());
@@ -144,15 +144,15 @@ public enum TableProperty {
         public Object getPTableValue(PTable table) {
             return table.getAutoPartitionSeqName();
         }  
-	},
-	
-	APPEND_ONLY_SCHEMA(PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, true, false) {
+    },
+
+    APPEND_ONLY_SCHEMA(PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, true, false) {
         @Override
         public Object getPTableValue(PTable table) {
             return table.isAppendOnlySchema();
         }
     },
-    GUIDE_POSTS_WIDTH(PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH, true, false, false) {
+    GUIDE_POSTS_WIDTH(PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH, true, false, false, false) {
         @Override
         public Object getValue(Object value) {
             return value == null ? null : ((Number) value).longValue();
@@ -162,30 +162,30 @@ public enum TableProperty {
         public Object getPTableValue(PTable table) {
             return null;
         }       
-	    
-	},
-	
-	COLUMN_ENCODED_BYTES(PhoenixDatabaseMetaData.ENCODING_SCHEME, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, false, false, false) {
-	    @Override
+
+    },
+
+    COLUMN_ENCODED_BYTES(PhoenixDatabaseMetaData.ENCODING_SCHEME, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, false, false, false) {
+        @Override
         public Object getValue(Object value) {
-	        if (value instanceof String) {
-	            String strValue = (String) value;
-	            if ("NONE".equalsIgnoreCase(strValue)) {
-	                return (byte)0;
-	            } 
-	        } else {
-	            return value == null ? null : ((Number) value).byteValue();
-	        }
-	        return value;
-	    }
-
-		@Override
-		public Object getPTableValue(PTable table) {
-			return table.getEncodingScheme();
-		}	
-	    
-	},
-    
+            if (value instanceof String) {
+                String strValue = (String) value;
+                if ("NONE".equalsIgnoreCase(strValue)) {
+                    return (byte)0;
+                } 
+            } else {
+                return value == null ? null : ((Number) value).byteValue();
+            }
+            return value;
+        }
+
+        @Override
+        public Object getPTableValue(PTable table) {
+            return table.getEncodingScheme();
+        }
+
+    },
+
     IMMUTABLE_STORAGE_SCHEME(PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, false, false) {
         @Override
         public ImmutableStorageScheme getValue(Object value) {
@@ -203,7 +203,7 @@ public enum TableProperty {
         public Object getPTableValue(PTable table) {
             return table.getImmutableStorageScheme();
         }   
-        
+
     },
 
     USE_STATS_FOR_PARALLELIZATION(PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION, true, true, true) {
@@ -224,81 +224,91 @@ public enum TableProperty {
         }
     }
     ;
-	
-	private final String propertyName;
-	private final SQLExceptionCode colFamSpecifiedException;
-	private final boolean isMutable; // whether or not a property can be changed through statements like ALTER TABLE.
-	private final SQLExceptionCode mutatingImmutablePropException;
-	private final boolean isValidOnView;
-	private final boolean isMutableOnView;
-
-	private TableProperty(String propertyName, boolean isMutable, boolean isValidOnView, boolean isMutableOnView) {
-		this(propertyName, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, isMutable, CANNOT_ALTER_PROPERTY, isValidOnView, isMutableOnView);
-	}
-
-	private TableProperty(String propertyName, SQLExceptionCode colFamilySpecifiedException, boolean isMutable, boolean isValidOnView, boolean isMutableOnView) {
-		this(propertyName, colFamilySpecifiedException, isMutable, CANNOT_ALTER_PROPERTY, isValidOnView, isMutableOnView);
-	}
-
-	private TableProperty(String propertyName, boolean isMutable, boolean isValidOnView, boolean isMutableOnView, SQLExceptionCode isMutatingException) {
-		this(propertyName, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, isMutable, isMutatingException, isValidOnView, isMutableOnView);
-	}
-
-	private TableProperty(String propertyName, SQLExceptionCode colFamSpecifiedException, boolean isMutable, SQLExceptionCode mutatingException, boolean isValidOnView, boolean isMutableOnView) {
-		this.propertyName = propertyName;
-		this.colFamSpecifiedException = colFamSpecifiedException;
-		this.isMutable = isMutable;
-		this.mutatingImmutablePropException = mutatingException;
-		this.isValidOnView = isValidOnView;
-		this.isMutableOnView = isMutableOnView;
-	}
-
-	public static boolean isPhoenixTableProperty(String property) {
-		try {
-			TableProperty.valueOf(property);
-		} catch (IllegalArgumentException e) {
-			return false;
-		}
-		return true;
-	}
-
-	public Object getValue(Object value) {
-	    return value;
-	}
-	
+
+    private final String propertyName;
+    private final SQLExceptionCode colFamSpecifiedException;
+    private final boolean isMutable; // whether or not a property can be changed through statements like ALTER TABLE.
+    private final SQLExceptionCode mutatingImmutablePropException;
+    private final boolean isValidOnView;
+    private final boolean isMutableOnView;
+    private final boolean propagateToViews;
+
+    private TableProperty(String propertyName, boolean isMutable, boolean isValidOnView, boolean isMutableOnView) {
+        this(propertyName, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, isMutable, CANNOT_ALTER_PROPERTY, isValidOnView, isMutableOnView, true);
+    }
+
+    private TableProperty(String propertyName, boolean isMutable, boolean isValidOnView, boolean isMutableOnView, boolean propagateToViews) {
+        this(propertyName, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, isMutable, CANNOT_ALTER_PROPERTY, isValidOnView, isMutableOnView, propagateToViews);
+    }
+
+    private TableProperty(String propertyName, SQLExceptionCode colFamilySpecifiedException, boolean isMutable, boolean isValidOnView, boolean isMutableOnView) {
+        this(propertyName, colFamilySpecifiedException, isMutable, CANNOT_ALTER_PROPERTY, isValidOnView, isMutableOnView, true);
+    }
+
+    private TableProperty(String propertyName, boolean isMutable, boolean isValidOnView, boolean isMutableOnView, SQLExceptionCode isMutatingException) {
+        this(propertyName, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, isMutable, isMutatingException, isValidOnView, isMutableOnView, true);
+    }
+
+    private TableProperty(String propertyName, SQLExceptionCode colFamSpecifiedException, boolean isMutable, SQLExceptionCode mutatingException, boolean isValidOnView, boolean isMutableOnView) {
+        this(propertyName, colFamSpecifiedException, isMutable, mutatingException, isValidOnView, isMutableOnView, true);
+    }
+
+    private TableProperty(String propertyName, SQLExceptionCode colFamSpecifiedException, boolean isMutable, SQLExceptionCode mutatingException, boolean isValidOnView, boolean isMutableOnView, boolean propagateToViews) {
+        this.propertyName = propertyName;
+        this.colFamSpecifiedException = colFamSpecifiedException;
+        this.isMutable = isMutable;
+        this.mutatingImmutablePropException = mutatingException;
+        this.isValidOnView = isValidOnView;
+        this.isMutableOnView = isMutableOnView;
+        this.propagateToViews = propagateToViews;
+    }
+
+    public static boolean isPhoenixTableProperty(String property) {
+        try {
+            TableProperty.valueOf(property);
+        } catch (IllegalArgumentException e) {
+            return false;
+        }
+        return true;
+    }
+
+    public Object getValue(Object value) {
+        return value;
+    }
+
     public Object getValue(Map<String, Object> props) {
         return getValue(props.get(this.toString()));
     }
-    
-	// isQualified is true if column family name is specified in property name
-	public void validate(boolean isMutating, boolean isQualified, PTableType tableType) throws SQLException {
-		checkForColumnFamily(isQualified);
-		checkIfApplicableForView(tableType);
-		checkForMutability(isMutating,tableType);
-	}
-
-	private void checkForColumnFamily(boolean isQualified) throws SQLException {
-		if (isQualified) {
-			throw new SQLExceptionInfo.Builder(colFamSpecifiedException).setMessage(". Property: " + propertyName).build().buildException();
-		}
-	}
-
-	private void checkForMutability(boolean isMutating, PTableType tableType) throws SQLException {
-		if (isMutating && !isMutable) {
-			throw new SQLExceptionInfo.Builder(mutatingImmutablePropException).setMessage(". Property: " + propertyName).build().buildException();
-		}
-		if (isMutating && tableType == PTableType.VIEW && !isMutableOnView) {
-			throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_TABLE_PROPERTY_ON_VIEW).setMessage(". Property: " + propertyName).build().buildException();
-		}
-	}
-
-	private void checkIfApplicableForView(PTableType tableType)
-			throws SQLException {
-		if (tableType == PTableType.VIEW && !isValidOnView) {
-			throw new SQLExceptionInfo.Builder(
-					VIEW_WITH_PROPERTIES).setMessage("Property: " + propertyName).build().buildException();
-		}
-	}
+
+    // isQualified is true if column family name is specified in property name
+    public void validate(boolean isMutating, boolean isQualified, PTableType tableType) throws SQLException {
+        checkForColumnFamily(isQualified);
+        checkIfApplicableForView(tableType);
+        checkForMutability(isMutating,tableType);
+    }
+
+    private void checkForColumnFamily(boolean isQualified) throws SQLException {
+        if (isQualified) {
+            throw new SQLExceptionInfo.Builder(colFamSpecifiedException).setMessage(". Property: " + propertyName).build().buildException();
+        }
+    }
+
+    private void checkForMutability(boolean isMutating, PTableType tableType) throws SQLException {
+        if (isMutating && !isMutable) {
+            throw new SQLExceptionInfo.Builder(mutatingImmutablePropException).setMessage(". Property: " + propertyName).build().buildException();
+        }
+        if (isMutating && tableType == PTableType.VIEW && !isMutableOnView) {
+            throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_TABLE_PROPERTY_ON_VIEW).setMessage(". Property: " + propertyName).build().buildException();
+        }
+    }
+
+    private void checkIfApplicableForView(PTableType tableType)
+            throws SQLException {
+        if (tableType == PTableType.VIEW && !isValidOnView) {
+            throw new SQLExceptionInfo.Builder(
+                    VIEW_WITH_PROPERTIES).setMessage("Property: " + propertyName).build().buildException();
+        }
+    }
 
     public String getPropertyName() {
         return propertyName;
@@ -315,7 +325,11 @@ public enum TableProperty {
     public boolean isMutableOnView() {
         return isMutableOnView;
     }
-    
+
+    public boolean propagateToViews() {
+        return propagateToViews;
+    }
+
     abstract public Object getPTableValue(PTable table);
-    
+
 }