You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by td...@apache.org on 2015/11/24 06:24:22 UTC
[1/9] phoenix git commit: PHOENIX-1674 Snapshot isolation transaction
support through Tephra (James Taylor, Thomas D'Silva)
Repository: phoenix
Updated Branches:
refs/heads/master 76a8d1804 -> cc9929b5c
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java
new file mode 100644
index 0000000..83d0b42
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema;
+
+public class PTableRef {
+ private final PTable table;
+ private final int estSize;
+ private volatile long lastAccessTime;
+ // timestamp (scn or txn timestamp at which rpc to fetch the table was made)
+ private long resolvedTimeStamp;
+
+ public PTableRef(PTable table, long lastAccessTime, int estSize, long resolvedTime) {
+ this.table = table;
+ this.lastAccessTime = lastAccessTime;
+ this.estSize = estSize;
+ this.resolvedTimeStamp = resolvedTime;
+ }
+
+ public PTableRef(PTable table, long lastAccessTime, long resolvedTime) {
+ this (table, lastAccessTime, table.getEstimatedSize(), resolvedTime);
+ }
+
+ public PTableRef(PTableRef tableRef) {
+ this (tableRef.table, tableRef.lastAccessTime, tableRef.estSize, tableRef.resolvedTimeStamp);
+ }
+
+ public PTable getTable() {
+ return table;
+ }
+
+ public long getResolvedTimeStamp() {
+ return resolvedTimeStamp;
+ }
+
+ public int getEstSize() {
+ return estSize;
+ }
+
+ public long getLastAccessTime() {
+ return lastAccessTime;
+ }
+
+ public void setLastAccessTime(long lastAccessTime) {
+ this.lastAccessTime = lastAccessTime;
+ }
+
+ public void setResolvedTimeStamp(long resolvedTimeStamp) {
+ this.resolvedTimeStamp = resolvedTimeStamp;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
index c429ac4..091b929 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
@@ -46,7 +46,10 @@ public enum TableProperty {
TTL(HColumnDescriptor.TTL, COLUMN_FAMILY_NOT_ALLOWED_FOR_TTL, true, CANNOT_ALTER_PROPERTY, false),
- STORE_NULLS(PhoenixDatabaseMetaData.STORE_NULLS, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, false);
+ STORE_NULLS(PhoenixDatabaseMetaData.STORE_NULLS, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, false),
+
+ TRANSACTIONAL(PhoenixDatabaseMetaData.TRANSACTIONAL, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, false),
+ ;
private final String propertyName;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
index 78b00fa..26cbbc3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
@@ -17,6 +17,8 @@
*/
package org.apache.phoenix.schema;
+import java.util.Objects;
+
import org.apache.hadoop.hbase.HConstants;
import org.apache.phoenix.compile.TupleProjectionCompiler;
import org.apache.phoenix.query.QueryConstants;
@@ -38,6 +40,10 @@ public class TableRef {
this(tableRef.alias, tableRef.table, timeStamp, tableRef.hasDynamicCols);
}
+ public TableRef(TableRef tableRef, String alias) {
+ this(alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.hasDynamicCols);
+ }
+
public TableRef(PTable table) {
this(null, table, HConstants.LATEST_TIMESTAMP, false);
}
@@ -101,7 +107,7 @@ public class TableRef {
public int hashCode() {
final int prime = 31;
int result = alias == null ? 0 : alias.hashCode();
- result = prime * result + this.table.getName().getString().hashCode();
+ result = prime * result + ( this.table.getName()!=null ? this.table.getName().hashCode() : 0);
return result;
}
@@ -111,8 +117,8 @@ public class TableRef {
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
TableRef other = (TableRef)obj;
- if ((alias == null && other.alias != null) || (alias != null && !alias.equals(other.alias))) return false;
- if (!table.getName().getString().equals(other.table.getName().getString())) return false;
+ if (!Objects.equals(alias, other.alias)) return false;
+ if (!Objects.equals(table.getName(), other.table.getName())) return false;
return true;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
index 159e0c9..c39df2c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricsRecord;
import org.apache.hadoop.metrics2.MetricsSink;
import org.apache.hadoop.metrics2.MetricsTag;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.metrics.MetricInfo;
import org.apache.phoenix.metrics.Metrics;
import org.apache.phoenix.query.QueryServices;
@@ -175,7 +176,13 @@ public class PhoenixMetricsSink implements MetricsSink {
TAG_COUNT + " smallint, " +
ANNOTATION_COUNT + " smallint" +
" CONSTRAINT pk PRIMARY KEY (" + TRACE.columnName + ", "
- + PARENT.columnName + ", " + SPAN.columnName + "))\n";
+ + PARENT.columnName + ", " + SPAN.columnName + "))\n" +
+ // We have a config parameter that can be set so that tables are
+ // transactional by default. If that's set, we still don't want these system
+ // tables created as transactional tables, make these table non
+ // transactional
+ PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
+;
PreparedStatement stmt = conn.prepareStatement(ddl);
stmt.execute();
this.table = table;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 3bf6f23..f361fb9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -30,6 +30,7 @@ import java.util.Map;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Mutation;
@@ -85,6 +86,8 @@ import org.apache.phoenix.schema.types.PDecimal;
import org.apache.phoenix.schema.types.PVarbinary;
import org.apache.phoenix.schema.types.PVarchar;
+import co.cask.tephra.TxConstants;
+
import com.google.common.collect.Lists;
public class IndexUtil {
@@ -209,13 +212,33 @@ public class IndexUtil {
.getLength()) == 0);
}
+ public static List<Delete> generateDeleteIndexData(final PTable table, PTable index,
+ List<Delete> dataMutations, ImmutableBytesWritable ptr, final KeyValueBuilder kvBuilder, PhoenixConnection connection)
+ throws SQLException {
+ try {
+ IndexMaintainer maintainer = index.getIndexMaintainer(table, connection);
+ List<Delete> indexMutations = Lists.newArrayListWithExpectedSize(dataMutations.size());
+ for (final Mutation dataMutation : dataMutations) {
+ long ts = MetaDataUtil.getClientTimeStamp(dataMutation);
+ ptr.set(dataMutation.getRow());
+ Delete delete = maintainer.buildDeleteMutation(kvBuilder, ptr, ts);
+ delete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, dataMutation.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY));
+ indexMutations.add(delete);
+ }
+ return indexMutations;
+ } catch (IOException e) {
+ throw new SQLException(e);
+ }
+ }
+
public static List<Mutation> generateIndexData(final PTable table, PTable index,
- List<Mutation> dataMutations, ImmutableBytesWritable ptr, final KeyValueBuilder kvBuilder, PhoenixConnection connection)
+ List<Mutation> dataMutations, final KeyValueBuilder kvBuilder, PhoenixConnection connection)
throws SQLException {
try {
+ final ImmutableBytesWritable ptr = new ImmutableBytesWritable();
IndexMaintainer maintainer = index.getIndexMaintainer(table, connection);
List<Mutation> indexMutations = Lists.newArrayListWithExpectedSize(dataMutations.size());
- for (final Mutation dataMutation : dataMutations) {
+ for (final Mutation dataMutation : dataMutations) {
long ts = MetaDataUtil.getClientTimeStamp(dataMutation);
ptr.set(dataMutation.getRow());
/*
@@ -235,7 +258,7 @@ public class IndexUtil {
}
@Override
- public ImmutableBytesPtr getLatestValue(ColumnReference ref) {
+ public ImmutableBytesWritable getLatestValue(ColumnReference ref) {
// Always return null for our empty key value, as this will cause the index
// maintainer to always treat this Put as a new row.
if (isEmptyKeyValue(table, ref)) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index ee73a58..47d3bc3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -272,6 +272,13 @@ public class MetaDataUtil {
return false;
}
+ public static boolean isTransactional(Mutation m, KeyValueBuilder builder, ImmutableBytesWritable ptr) {
+ if (getMutationValue(m, PhoenixDatabaseMetaData.TRANSACTIONAL_BYTES, builder, ptr)) {
+ return Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(ptr));
+ }
+ return false;
+ }
+
public static boolean isSalted(Mutation m, KeyValueBuilder builder, ImmutableBytesWritable ptr) {
return MetaDataUtil.getMutationValue(m, PhoenixDatabaseMetaData.SALT_BUCKETS_BYTES, builder, ptr);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
index 183bfa7..912dd03 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -511,7 +511,7 @@ public class ScanUtil {
}
}
- public static ScanRanges newScanRanges(List<Mutation> mutations) throws SQLException {
+ public static ScanRanges newScanRanges(List<? extends Mutation> mutations) throws SQLException {
List<KeyRange> keys = Lists.newArrayListWithExpectedSize(mutations.size());
for (Mutation m : mutations) {
keys.add(PVarbinary.INSTANCE.getKeyRange(m.getRow()));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
index 5414d4f..1693600 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
@@ -30,7 +30,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Properties;
+import java.util.Map;
import javax.annotation.Nullable;
@@ -51,13 +51,13 @@ import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PColumnFamily;
import org.apache.phoenix.schema.PDatum;
-import org.apache.phoenix.schema.PMetaData;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.RowKeySchema;
import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder;
import org.apache.phoenix.schema.SaltingUtil;
import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.TableProperty;
import org.apache.phoenix.schema.ValueSchema.Field;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PVarbinary;
@@ -262,6 +262,9 @@ public class SchemaUtil {
public static String getMetaDataEntityName(String schemaName, String tableName, String familyName, String columnName) {
if ((schemaName == null || schemaName.isEmpty()) && (tableName == null || tableName.isEmpty())) {
+ if (columnName == null || columnName.isEmpty()) {
+ return familyName;
+ }
return getName(familyName, columnName, false);
}
if ((familyName == null || familyName.isEmpty()) && (columnName == null || columnName.isEmpty())) {
@@ -390,6 +393,11 @@ public class SchemaUtil {
return families.isEmpty() ? table.getDefaultFamilyName() == null ? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES : table.getDefaultFamilyName().getBytes() : families.get(0).getName().getBytes();
}
+ public static String getEmptyColumnFamilyAsString(PTable table) {
+ List<PColumnFamily> families = table.getColumnFamilies();
+ return families.isEmpty() ? table.getDefaultFamilyName() == null ? QueryConstants.DEFAULT_COLUMN_FAMILY : table.getDefaultFamilyName().getString() : families.get(0).getName().getString();
+ }
+
public static ImmutableBytesPtr getEmptyColumnFamilyPtr(PTable table) {
List<PColumnFamily> families = table.getColumnFamilies();
return families.isEmpty() ? table.getDefaultFamilyName() == null ? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES_PTR : table.getDefaultFamilyName().getBytesPtr() : families.get(0)
@@ -517,15 +525,10 @@ public class SchemaUtil {
}
protected static PhoenixConnection addMetaDataColumn(PhoenixConnection conn, long scn, String columnDef) throws SQLException {
- String url = conn.getURL();
- Properties props = conn.getClientInfo();
- PMetaData metaData = conn.getMetaDataCache();
- props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(scn));
PhoenixConnection metaConnection = null;
-
Statement stmt = null;
try {
- metaConnection = new PhoenixConnection(conn.getQueryServices(), url, props, metaData);
+ metaConnection = new PhoenixConnection(conn.getQueryServices(), conn, scn);
try {
stmt = metaConnection.createStatement();
stmt.executeUpdate("ALTER TABLE SYSTEM.\"TABLE\" ADD IF NOT EXISTS " + columnDef);
@@ -711,6 +714,16 @@ public class SchemaUtil {
checkArgument(!isNullOrEmpty(columnName), "Column name cannot be null or empty");
return columnFamilyName == null ? ("\"" + columnName + "\"") : ("\"" + columnFamilyName + "\"" + QueryConstants.NAME_SEPARATOR + "\"" + columnName + "\"");
}
+
+ public static boolean hasHTableDescriptorProps(Map<String, Object> tableProps) {
+ int pTablePropCount = 0;
+ for (String prop : tableProps.keySet()) {
+ if (TableProperty.isPhoenixTableProperty(prop)) {
+ pTablePropCount++;
+ }
+ }
+ return tableProps.size() - pTablePropCount > 0;
+ }
/**
* Replaces all occurrences of {@link #ESCAPE_CHARACTER} with an empty character.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java
index a51dc84..679c7a1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.HTablePool;
+import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.phoenix.exception.PhoenixIOException;
import org.apache.phoenix.exception.SQLExceptionCode;
@@ -66,7 +67,9 @@ public class ServerUtil {
} else if (t instanceof IOException) {
// If the IOException does not wrap any exception, then bubble it up.
Throwable cause = t.getCause();
- if (cause == null || cause instanceof IOException) {
+ if (cause instanceof RetriesExhaustedWithDetailsException)
+ return new DoNotRetryIOException(t.getMessage(), cause);
+ else if (cause == null || cause instanceof IOException) {
return (IOException) t;
}
// Else assume it's been wrapped, so throw as DoNotRetryIOException to prevent client hanging while retrying
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java
new file mode 100644
index 0000000..9f16e52
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import java.sql.SQLException;
+
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.exception.SQLExceptionInfo;
+import org.apache.phoenix.execute.MutationState;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.schema.PTable;
+
+import co.cask.tephra.TransactionConflictException;
+import co.cask.tephra.TransactionFailureException;
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.hbase11.TransactionAwareHTable;
+
+public class TransactionUtil {
+ private TransactionUtil() {
+ }
+
+ public static long convertToNanoseconds(long serverTimeStamp) {
+ return serverTimeStamp * TxConstants.MAX_TX_PER_MS;
+ }
+
+ public static long convertToMilliseconds(long serverTimeStamp) {
+ return serverTimeStamp / TxConstants.MAX_TX_PER_MS;
+ }
+
+ public static SQLException getTransactionFailureException(TransactionFailureException e) {
+ if (e instanceof TransactionConflictException) {
+ return new SQLExceptionInfo.Builder(SQLExceptionCode.TRANSACTION_CONFLICT_EXCEPTION)
+ .setMessage(e.getMessage())
+ .setRootCause(e)
+ .build().buildException();
+
+ }
+ return new SQLExceptionInfo.Builder(SQLExceptionCode.TRANSACTION_FAILED)
+ .setMessage(e.getMessage())
+ .setRootCause(e)
+ .build().buildException();
+ }
+
+ public static TransactionAwareHTable getTransactionAwareHTable(HTableInterface htable, PTable table) {
+ // Conflict detection is not needed for tables with write-once/append-only data
+ return new TransactionAwareHTable(htable, table.isImmutableRows() ? TxConstants.ConflictDetection.NONE : TxConstants.ConflictDetection.ROW);
+ }
+
+ // we resolve transactional tables at the txn read pointer
+ public static long getResolvedTimestamp(PhoenixConnection connection, boolean isTransactional, long defaultResolvedTimestamp) {
+ MutationState mutationState = connection.getMutationState();
+ Long scn = connection.getSCN();
+ return scn != null ? scn : (isTransactional && mutationState.isTransactionStarted()) ? convertToMilliseconds(mutationState.getReadPointer()) : defaultResolvedTimestamp;
+ }
+
+ public static long getResolvedTime(PhoenixConnection connection, MetaDataMutationResult result) {
+ PTable table = result.getTable();
+ boolean isTransactional = table!=null && table.isTransactional();
+ return getResolvedTimestamp(connection, isTransactional, result.getMutationTime());
+ }
+
+ public static long getResolvedTimestamp(PhoenixConnection connection, MetaDataMutationResult result) {
+ PTable table = result.getTable();
+ MutationState mutationState = connection.getMutationState();
+ boolean txInProgress = table != null && table.isTransactional() && mutationState.isTransactionStarted();
+ return txInProgress ? convertToMilliseconds(mutationState.getReadPointer()) : result.getMutationTime();
+ }
+
+ public static Long getTableTimestamp(PhoenixConnection connection, boolean transactional) throws SQLException {
+ Long timestamp = null;
+ if (!transactional) {
+ return timestamp;
+ }
+ MutationState mutationState = connection.getMutationState();
+ // we need to burn a txn so that we are sure the txn read pointer is close to wall clock time
+ if (!mutationState.isTransactionStarted()) {
+ mutationState.startTransaction();
+ connection.commit();
+ }
+ else {
+ connection.commit();
+ }
+ mutationState.startTransaction();
+ timestamp = convertToMilliseconds(mutationState.getReadPointer());
+ connection.commit();
+ return timestamp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
index 23eb147..9da1a67 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
@@ -1482,7 +1482,7 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest {
statement.execute("create local index my_idx on example (fn) DEFAULT_COLUMN_FAMILY='F'");
fail();
} catch (SQLException e) {
- assertEquals(SQLExceptionCode.VIEW_WITH_PROPERTIES.getErrorCode(),e.getErrorCode());
+ assertEquals(SQLExceptionCode.DEFAULT_COLUMN_FAMILY_ON_SHARED_TABLE.getErrorCode(),e.getErrorCode());
}
statement.execute("create local index my_idx on example (fn)");
} finally {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/compile/ViewCompilerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/ViewCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/ViewCompilerTest.java
index 1d95058..959031f 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/ViewCompilerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/ViewCompilerTest.java
@@ -31,20 +31,20 @@ import org.apache.phoenix.query.BaseConnectionlessQueryTest;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.ViewType;
-import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.util.PropertiesUtil;
import org.junit.Test;
public class ViewCompilerTest extends BaseConnectionlessQueryTest {
@Test
public void testViewTypeCalculation() throws Exception {
- assertViewType(new String[] {
+ assertViewType(new String[] {"V1","V2","V3","V4"}, new String[] {
"CREATE VIEW v1 AS SELECT * FROM t WHERE k1 = 1 AND k2 = 'foo'",
"CREATE VIEW v2 AS SELECT * FROM t WHERE k2 = 'foo'",
"CREATE VIEW v3 AS SELECT * FROM t WHERE v = 'bar'||'bas'",
"CREATE VIEW v4 AS SELECT * FROM t WHERE 'bar'=v and 5+3/2 = k1",
}, ViewType.UPDATABLE);
- assertViewType(new String[] {
+ assertViewType(new String[] {"V1","V2","V3","V4"}, new String[] {
"CREATE VIEW v1 AS SELECT * FROM t WHERE k1 < 1 AND k2 = 'foo'",
"CREATE VIEW v2 AS SELECT * FROM t WHERE substr(k2,0,3) = 'foo'",
"CREATE VIEW v3 AS SELECT * FROM t WHERE v = TO_CHAR(CURRENT_DATE())",
@@ -52,28 +52,27 @@ public class ViewCompilerTest extends BaseConnectionlessQueryTest {
}, ViewType.READ_ONLY);
}
- public void assertViewType(String[] views, ViewType viewType) throws Exception {
+ public void assertViewType(String[] viewNames, String[] viewDDLs, ViewType viewType) throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
PhoenixConnection conn = DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class);
String ct = "CREATE TABLE t (k1 INTEGER NOT NULL, k2 VARCHAR, v VARCHAR, CONSTRAINT pk PRIMARY KEY (k1,k2))";
conn.createStatement().execute(ct);
- for (String view : views) {
- conn.createStatement().execute(view);
+ for (String viewDDL : viewDDLs) {
+ conn.createStatement().execute(viewDDL);
}
StringBuilder buf = new StringBuilder();
int count = 0;
- for (PTable table : conn.getMetaDataCache()) {
- if (table.getType() == PTableType.VIEW) {
- assertEquals(viewType, table.getViewType());
- conn.createStatement().execute("DROP VIEW " + table.getName().getString());
- buf.append(' ');
- buf.append(table.getName().getString());
- count++;
- }
+ for (String view : viewNames) {
+ PTable table = conn.getTable(new PTableKey(null, view));
+ assertEquals(viewType, table.getViewType());
+ conn.createStatement().execute("DROP VIEW " + table.getName().getString());
+ buf.append(' ');
+ buf.append(table.getName().getString());
+ count++;
}
- assertEquals("Expected " + views.length + ", but got " + count + ":"+ buf.toString(), views.length, count);
+ assertEquals("Expected " + viewDDLs.length + ", but got " + count + ":"+ buf.toString(), viewDDLs.length, count);
}
@Test
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
index 334ce8c..9d5e17c 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
@@ -233,7 +233,7 @@ public class CorrelatePlanTest {
PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM,
null, null, columns, null, null, Collections.<PTable>emptyList(),
false, Collections.<PName>emptyList(), null, null, false, false, false, null,
- null, null, true);
+ null, null, true, false);
TableRef sourceTable = new TableRef(pTable);
List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList();
for (PColumn column : sourceTable.getTable().getColumns()) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
index d142672..a0aca4d 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.SortedMap;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.compile.QueryPlan;
import org.apache.phoenix.end2end.Shadower;
@@ -670,7 +671,7 @@ public class SkipScanBigFilterTest extends BaseConnectionlessQueryTest {
return table.getTimeStamp()+1;
}
});
- conn.unwrap(PhoenixConnection.class).addTable(tableWithStats);
+ conn.unwrap(PhoenixConnection.class).addTable(tableWithStats, System.currentTimeMillis());
String query = "SELECT count(1) cnt,\n" +
" coalesce(SUM(impressions), 0.0) AS \"impressions\",\n" +
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/CoveredIndexCodecForTesting.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/CoveredIndexCodecForTesting.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/CoveredIndexCodecForTesting.java
index b0fae03..7d31516 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/CoveredIndexCodecForTesting.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/CoveredIndexCodecForTesting.java
@@ -25,49 +25,46 @@ import java.util.List;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.phoenix.hbase.index.builder.BaseIndexCodec;
-import org.apache.phoenix.hbase.index.covered.IndexCodec;
-import org.apache.phoenix.hbase.index.covered.IndexUpdate;
-import org.apache.phoenix.hbase.index.covered.TableState;
/**
- * An {@link IndexCodec} for testing that allow you to specify the index updates/deletes, regardless
- * of the current tables' state.
+ * An {@link IndexCodec} for testing that allow you to specify the index updates/deletes, regardless of the current
+ * tables' state.
*/
public class CoveredIndexCodecForTesting extends BaseIndexCodec {
- private List<IndexUpdate> deletes = new ArrayList<IndexUpdate>();
- private List<IndexUpdate> updates = new ArrayList<IndexUpdate>();
+ private List<IndexUpdate> deletes = new ArrayList<IndexUpdate>();
+ private List<IndexUpdate> updates = new ArrayList<IndexUpdate>();
- public void addIndexDelete(IndexUpdate... deletes) {
- this.deletes.addAll(Arrays.asList(deletes));
- }
-
- public void addIndexUpserts(IndexUpdate... updates) {
- this.updates.addAll(Arrays.asList(updates));
- }
+ public void addIndexDelete(IndexUpdate... deletes) {
+ this.deletes.addAll(Arrays.asList(deletes));
+ }
- public void clear() {
- this.deletes.clear();
- this.updates.clear();
- }
-
- @Override
- public Iterable<IndexUpdate> getIndexDeletes(TableState state, IndexMetaData context) {
- return this.deletes;
- }
+ public void addIndexUpserts(IndexUpdate... updates) {
+ this.updates.addAll(Arrays.asList(updates));
+ }
- @Override
- public Iterable<IndexUpdate> getIndexUpserts(TableState state, IndexMetaData context) {
- return this.updates;
- }
+ public void clear() {
+ this.deletes.clear();
+ this.updates.clear();
+ }
- @Override
- public void initialize(RegionCoprocessorEnvironment env) throws IOException {
- // noop
- }
+ @Override
+ public Iterable<IndexUpdate> getIndexDeletes(TableState state, IndexMetaData context) {
+ return this.deletes;
+ }
- @Override
- public boolean isEnabled(Mutation m) {
- return true;
- }
+ @Override
+ public Iterable<IndexUpdate> getIndexUpserts(TableState state, IndexMetaData context) {
+ return this.updates;
+ }
+
+ @Override
+ public void initialize(RegionCoprocessorEnvironment env) throws IOException {
+ // noop
+ }
+
+ @Override
+ public boolean isEnabled(Mutation m) {
+ return true;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java
index 27ed370..e941ceb 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java
@@ -73,7 +73,7 @@ public class IndexMaintainerTest extends BaseConnectionlessQueryTest {
return new ValueGetter() {
@Override
- public ImmutableBytesPtr getLatestValue(ColumnReference ref) {
+ public ImmutableBytesWritable getLatestValue(ColumnReference ref) {
return new ImmutableBytesPtr(valueMap.get(ref));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
index d4956ee..f6df5c1 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
@@ -88,9 +88,9 @@ public class PhoenixTestDriver extends PhoenixEmbeddedDriver {
if (connectionQueryServices != null) { return connectionQueryServices; }
ConnectionInfo connInfo = ConnectionInfo.create(url);
if (connInfo.isConnectionless()) {
- connectionQueryServices = new ConnectionlessQueryServicesImpl(queryServices, connInfo);
+ connectionQueryServices = new ConnectionlessQueryServicesImpl(queryServices, connInfo, info);
} else {
- connectionQueryServices = new ConnectionQueryServicesTestImpl(queryServices, connInfo);
+ connectionQueryServices = new ConnectionQueryServicesTestImpl(queryServices, connInfo, info);
}
connectionQueryServices.init(url, info);
return connectionQueryServices;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index 8ad4f12..fa5df31 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -82,11 +82,13 @@ import static org.apache.phoenix.util.TestUtil.STABLE_NAME;
import static org.apache.phoenix.util.TestUtil.TABLE_WITH_ARRAY;
import static org.apache.phoenix.util.TestUtil.TABLE_WITH_SALTING;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.apache.phoenix.util.TestUtil.TRANSACTIONAL_DATA_TABLE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
@@ -106,6 +108,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
@@ -130,6 +133,7 @@ import org.apache.phoenix.end2end.BaseClientManagedTimeIT;
import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver;
+import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
import org.apache.phoenix.jdbc.PhoenixTestDriver;
import org.apache.phoenix.schema.NewerTableAlreadyExistsException;
import org.apache.phoenix.schema.PTableType;
@@ -143,12 +147,28 @@ import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
+import org.apache.twill.discovery.DiscoveryService;
+import org.apache.twill.discovery.ZKDiscoveryService;
+import org.apache.twill.internal.utils.Networks;
+import org.apache.twill.zookeeper.RetryStrategies;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.apache.twill.zookeeper.ZKClientServices;
+import org.apache.twill.zookeeper.ZKClients;
+import org.junit.ClassRule;
+import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import co.cask.tephra.TransactionManager;
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.distributed.TransactionService;
+import co.cask.tephra.metrics.TxMetricsCollector;
+import co.cask.tephra.persist.InMemoryTransactionStateStorage;
+
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import com.google.inject.util.Providers;
/**
*
@@ -187,7 +207,13 @@ public abstract class BaseTest {
" CONSTRAINT pk PRIMARY KEY (varchar_pk, char_pk, int_pk, long_pk DESC, decimal_pk, date_pk)) ";
private static final Map<String,String> tableDDLMap;
private static final Logger logger = LoggerFactory.getLogger(BaseTest.class);
-
+ protected static final int DEFAULT_TXN_TIMEOUT_SECONDS = 30;
+ private static ZKClientService zkClient;
+ private static TransactionService txService;
+ protected static TransactionManager txManager;
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+
static {
ImmutableMap.Builder<String,String> builder = ImmutableMap.builder();
builder.put(ENTITY_HISTORY_TABLE_NAME,"create table " + ENTITY_HISTORY_TABLE_NAME +
@@ -401,48 +427,9 @@ public abstract class BaseTest {
builder.put("KVBigIntValueTest", "create table KVBigIntValueTest" +
" (pk integer not null primary key,\n" +
" kv bigint)\n");
- builder.put(INDEX_DATA_TABLE, "create table " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + INDEX_DATA_TABLE + "(" +
- " varchar_pk VARCHAR NOT NULL, " +
- " char_pk CHAR(6) NOT NULL, " +
- " int_pk INTEGER NOT NULL, "+
- " long_pk BIGINT NOT NULL, " +
- " decimal_pk DECIMAL(31, 10) NOT NULL, " +
- " date_pk DATE NOT NULL, " +
- " a.varchar_col1 VARCHAR, " +
- " a.char_col1 CHAR(10), " +
- " a.int_col1 INTEGER, " +
- " a.long_col1 BIGINT, " +
- " a.decimal_col1 DECIMAL(31, 10), " +
- " a.date1 DATE, " +
- " b.varchar_col2 VARCHAR, " +
- " b.char_col2 CHAR(10), " +
- " b.int_col2 INTEGER, " +
- " b.long_col2 BIGINT, " +
- " b.decimal_col2 DECIMAL(31, 10), " +
- " b.date2 DATE " +
- " CONSTRAINT pk PRIMARY KEY (varchar_pk, char_pk, int_pk, long_pk DESC, decimal_pk, date_pk)) " +
- "IMMUTABLE_ROWS=true");
- builder.put(MUTABLE_INDEX_DATA_TABLE, "create table " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE + "(" +
- " varchar_pk VARCHAR NOT NULL, " +
- " char_pk CHAR(6) NOT NULL, " +
- " int_pk INTEGER NOT NULL, "+
- " long_pk BIGINT NOT NULL, " +
- " decimal_pk DECIMAL(31, 10) NOT NULL, " +
- " date_pk DATE NOT NULL, " +
- " a.varchar_col1 VARCHAR, " +
- " a.char_col1 CHAR(10), " +
- " a.int_col1 INTEGER, " +
- " a.long_col1 BIGINT, " +
- " a.decimal_col1 DECIMAL(31, 10), " +
- " a.date1 DATE, " +
- " b.varchar_col2 VARCHAR, " +
- " b.char_col2 CHAR(10), " +
- " b.int_col2 INTEGER, " +
- " b.long_col2 BIGINT, " +
- " b.decimal_col2 DECIMAL(31, 10), " +
- " b.date2 DATE " +
- " CONSTRAINT pk PRIMARY KEY (varchar_pk, char_pk, int_pk, long_pk DESC, decimal_pk, date_pk)) "
- );
+ builder.put(INDEX_DATA_TABLE, "create table " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + INDEX_DATA_TABLE + TEST_TABLE_SCHEMA + "IMMUTABLE_ROWS=true");
+ builder.put(MUTABLE_INDEX_DATA_TABLE, "create table " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE + TEST_TABLE_SCHEMA);
+ builder.put(TRANSACTIONAL_DATA_TABLE, "create table " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + TRANSACTIONAL_DATA_TABLE + TEST_TABLE_SCHEMA + "TRANSACTIONAL=true");
builder.put("SumDoubleTest","create table SumDoubleTest" +
" (id varchar not null primary key, d DOUBLE, f FLOAT, ud UNSIGNED_DOUBLE, uf UNSIGNED_FLOAT, i integer, de decimal)");
builder.put(JOIN_ORDER_TABLE_FULL_NAME, "create table " + JOIN_ORDER_TABLE_FULL_NAME +
@@ -499,9 +486,9 @@ public abstract class BaseTest {
return conf.get(QueryServices.ZOOKEEPER_PORT_ATTRIB);
}
- private static String url;
+ protected static String url;
protected static PhoenixTestDriver driver;
- private static boolean clusterInitialized = false;
+ protected static boolean clusterInitialized = false;
private static HBaseTestingUtility utility;
protected static final Configuration config = HBaseConfiguration.create();
@@ -512,10 +499,53 @@ public abstract class BaseTest {
return url;
}
- protected static String checkClusterInitialized(ReadOnlyProps overrideProps) {
+ private static void teardownTxManager() throws SQLException {
+ try {
+ if (txService != null) txService.stopAndWait();
+ } finally {
+ try {
+ if (zkClient != null) zkClient.stopAndWait();
+ } finally {
+ txService = null;
+ zkClient = null;
+ }
+ }
+
+ }
+
+ protected static void setupTxManager() throws SQLException, IOException {
+ config.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
+ config.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times");
+ config.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1);
+ config.setInt(TxConstants.Service.CFG_DATA_TX_BIND_PORT, Networks.getRandomPort());
+ config.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());
+ config.setInt(TxConstants.Manager.CFG_TX_TIMEOUT, DEFAULT_TXN_TIMEOUT_SECONDS);
+
+ ConnectionInfo connInfo = ConnectionInfo.create(getUrl());
+ zkClient = ZKClientServices.delegate(
+ ZKClients.reWatchOnExpire(
+ ZKClients.retryOnFailure(
+ ZKClientService.Builder.of(connInfo.getZookeeperConnectionString())
+ .setSessionTimeout(config.getInt(HConstants.ZK_SESSION_TIMEOUT,
+ HConstants.DEFAULT_ZK_SESSION_TIMEOUT))
+ .build(),
+ RetryStrategies.exponentialDelay(500, 2000, TimeUnit.MILLISECONDS)
+ )
+ )
+ );
+ zkClient.startAndWait();
+
+ DiscoveryService discovery = new ZKDiscoveryService(zkClient);
+ txManager = new TransactionManager(config, new InMemoryTransactionStateStorage(), new TxMetricsCollector());
+ txService = new TransactionService(config, zkClient, discovery, Providers.of(txManager));
+ txService.startAndWait();
+ }
+
+ protected static String checkClusterInitialized(ReadOnlyProps overrideProps) throws Exception {
if (!clusterInitialized) {
url = setUpTestCluster(config, overrideProps);
clusterInitialized = true;
+ setupTxManager();
}
return url;
}
@@ -560,8 +590,12 @@ public abstract class BaseTest {
utility.shutdownMiniCluster();
}
} finally {
- utility = null;
- clusterInitialized = false;
+ try {
+ teardownTxManager();
+ } finally {
+ utility = null;
+ clusterInitialized = false;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
index 732563f..089c5f1 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
@@ -20,7 +20,6 @@ package org.apache.phoenix.query;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.sql.Connection;
@@ -150,7 +149,7 @@ public class ConnectionlessTest {
assertTrue(iterator.hasNext());
kv = iterator.next();
assertArrayEquals(expectedRowKey1, kv.getRow());
- assertNull(PVarchar.INSTANCE.toObject(kv.getValue()));
+ assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, PVarchar.INSTANCE.toObject(kv.getValue()));
}
private static void assertRow2(Iterator<KeyValue> iterator, byte[] expectedRowKey2) {
@@ -166,7 +165,7 @@ public class ConnectionlessTest {
assertTrue(iterator.hasNext());
kv = iterator.next();
assertArrayEquals(expectedRowKey2, kv.getRow());
- assertNull(PVarchar.INSTANCE.toObject(kv.getValue()));
+ assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, PVarchar.INSTANCE.toObject(kv.getValue()));
}
@Test
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java
index e4ac5d3..14b5f1f 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java
@@ -31,7 +31,7 @@ public class PMetaDataImplTest {
private static PMetaData addToTable(PMetaData metaData, String name, int size) throws SQLException {
PTable table = new PSizedTable(new PTableKey(null,name), size);
- return metaData.addTable(table);
+ return metaData.addTable(table, System.currentTimeMillis());
}
private static PMetaData removeFromTable(PMetaData metaData, String name) throws SQLException {
@@ -39,7 +39,7 @@ public class PMetaDataImplTest {
}
private static PTable getFromTable(PMetaData metaData, String name) throws TableNotFoundException {
- return metaData.getTable(new PTableKey(null,name));
+ return metaData.getTableRef(new PTableKey(null,name)).getTable();
}
private static void assertNames(PMetaData metaData, String... names) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index 0fb9473..451ce62 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -25,14 +25,17 @@ import static org.apache.phoenix.util.PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PAR
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.sql.Connection;
+import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
+import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
@@ -98,7 +101,7 @@ public class TestUtil {
public static final String DEFAULT_INDEX_TABLE_NAME = "I";
public static final String DEFAULT_DATA_TABLE_FULL_NAME = SchemaUtil.getTableName(DEFAULT_SCHEMA_NAME, "T");
public static final String DEFAULT_INDEX_TABLE_FULL_NAME = SchemaUtil.getTableName(DEFAULT_SCHEMA_NAME, "I");
-
+
private TestUtil() {
}
@@ -191,6 +194,7 @@ public class TestUtil {
public static final String INDEX_DATA_SCHEMA = "INDEX_TEST";
public static final String INDEX_DATA_TABLE = "INDEX_DATA_TABLE";
public static final String MUTABLE_INDEX_DATA_TABLE = "MUTABLE_INDEX_DATA_TABLE";
+ public static final String TRANSACTIONAL_DATA_TABLE = "TRANSACTIONAL_DATA_TABLE";
public static final String JOIN_SCHEMA = "Join";
public static final String JOIN_ORDER_TABLE = "OrderTable";
public static final String JOIN_CUSTOMER_TABLE = "CustomerTable";
@@ -582,4 +586,36 @@ public class TestUtil {
analyzeTable(conn, tableName);
conn.close();
}
+
+ public static void setRowKeyColumns(PreparedStatement stmt, int i) throws SQLException {
+ // insert row
+ stmt.setString(1, "varchar" + String.valueOf(i));
+ stmt.setString(2, "char" + String.valueOf(i));
+ stmt.setInt(3, i);
+ stmt.setLong(4, i);
+ stmt.setBigDecimal(5, new BigDecimal(i*0.5d));
+ Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * TestUtil.NUM_MILLIS_IN_DAY);
+ stmt.setDate(6, date);
+ }
+
+ public static void validateRowKeyColumns(ResultSet rs, int i) throws SQLException {
+ assertTrue(rs.next());
+ assertEquals(rs.getString(1), "varchar" + String.valueOf(i));
+ assertEquals(rs.getString(2), "char" + String.valueOf(i));
+ assertEquals(rs.getInt(3), i);
+ assertEquals(rs.getInt(4), i);
+ assertEquals(rs.getBigDecimal(5), new BigDecimal(i*0.5d));
+ Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * TestUtil.NUM_MILLIS_IN_DAY);
+ assertEquals(rs.getDate(6), date);
+ }
+
+ public static String getTableName(Boolean mutable, Boolean transactional) {
+ StringBuilder tableNameBuilder = new StringBuilder(DEFAULT_DATA_TABLE_NAME);
+ if (mutable!=null)
+ tableNameBuilder.append(mutable ? "_MUTABLE" : "_IMMUTABLE");
+ if (transactional!=null)
+ tableNameBuilder.append(transactional ? "_TXN" : "_NON_TXN");
+ return tableNameBuilder.toString();
+ }
}
+
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-flume/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-flume/pom.xml b/phoenix-flume/pom.xml
index 1ab3fe7..b371567 100644
--- a/phoenix-flume/pom.xml
+++ b/phoenix-flume/pom.xml
@@ -164,6 +164,13 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
</dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <type>test-jar</type>
+ <version>${tephra.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-pig/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-pig/pom.xml b/phoenix-pig/pom.xml
index 7eef250..53406ff 100644
--- a/phoenix-pig/pom.xml
+++ b/phoenix-pig/pom.xml
@@ -160,6 +160,13 @@
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <type>test-jar</type>
+ <version>${tephra.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-protocol/src/main/PTable.proto
----------------------------------------------------------------------
diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto
index 3257a70..3048a40 100644
--- a/phoenix-protocol/src/main/PTable.proto
+++ b/phoenix-protocol/src/main/PTable.proto
@@ -86,4 +86,5 @@ message PTable {
optional bool storeNulls = 24;
optional int32 baseColumnCount = 25;
optional bool rowKeyOrderOptimizable = 26;
+ optional bool transactional = 27;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-protocol/src/main/ServerCachingService.proto
----------------------------------------------------------------------
diff --git a/phoenix-protocol/src/main/ServerCachingService.proto b/phoenix-protocol/src/main/ServerCachingService.proto
index 63aeef9..dcbaabd 100644
--- a/phoenix-protocol/src/main/ServerCachingService.proto
+++ b/phoenix-protocol/src/main/ServerCachingService.proto
@@ -37,6 +37,7 @@ message AddServerCacheRequest {
required bytes cacheId = 2;
required ImmutableBytesWritable cachePtr = 3;
required ServerCacheFactory cacheFactory = 4;
+ optional bytes txState = 5;
}
message AddServerCacheResponse {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-server/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-server/pom.xml b/phoenix-server/pom.xml
index 89ea5b5..2584748 100644
--- a/phoenix-server/pom.xml
+++ b/phoenix-server/pom.xml
@@ -106,5 +106,12 @@
<artifactId>hbase-testing-util</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <type>test-jar</type>
+ <version>${tephra.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3eb1f7c..148bb51 100644
--- a/pom.xml
+++ b/pom.xml
@@ -98,7 +98,7 @@
<commons-logging.version>1.2</commons-logging.version>
<commons-csv.version>1.0</commons-csv.version>
<sqlline.version>1.1.8</sqlline.version>
- <guava.version>12.0.1</guava.version>
+ <guava.version>13.0.1</guava.version>
<jackson.version>1.8.8</jackson.version>
<flume.version>1.4.0</flume.version>
<findbugs.version>1.3.2</findbugs.version>
@@ -112,6 +112,7 @@
<joni.version>2.1.2</joni.version>
<calcite.version>1.5.0</calcite.version>
<jettyVersion>8.1.7.v20120910</jettyVersion>
+ <tephra.version>0.6.3</tephra.version>
<!-- Test Dependencies -->
<mockito-all.version>1.8.5</mockito-all.version>
@@ -126,7 +127,7 @@
<maven-dependency-plugin.version>2.1</maven-dependency-plugin.version>
<maven.assembly.version>2.5.2</maven.assembly.version>
<maven.rat.version>0.8</maven.rat.version>
-
+
<!-- Plugin options -->
<numForkedUT>3</numForkedUT>
<numForkedIT>5</numForkedIT>
[8/9] phoenix git commit: PHOENIX-1674 Snapshot isolation transaction
support through Tephra (James Taylor, Thomas D'Silva)
Posted by td...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
index b269212..8f5301c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
@@ -52,17 +52,28 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
protected final boolean localIndex;
private final String tableDDLOptions;
+ private final String tableName;
+ private final String indexName;
+ private final String fullTableName;
+ private final String fullIndexName;
- public MutableIndexIT(boolean localIndex) {
+ public MutableIndexIT(boolean localIndex, boolean transactional) {
this.localIndex = localIndex;
StringBuilder optionBuilder = new StringBuilder();
+ if (transactional) {
+ optionBuilder.append("TRANSACTIONAL=true");
+ }
this.tableDDLOptions = optionBuilder.toString();
+ this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + ( transactional ? "_TXN" : "");
+ this.indexName = "IDX" + ( transactional ? "_TXN" : "");
+ this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ this.fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
}
- @Parameters(name="localIndex = {0}")
+ @Parameters(name="localIndex = {0} , transactional = {1}")
public static Collection<Boolean[]> data() {
return Arrays.asList(new Boolean[][] {
- { false }, {true}
+ { false, false }, { false, true }, { true, false }, { true, true }
});
}
@@ -71,12 +82,6 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
createMultiCFTestTable(fullTableName, tableDDLOptions);
populateMultiCFTestTable(fullTableName);
PreparedStatement stmt = conn.prepareStatement("CREATE " + (localIndex ? " LOCAL " : "") + " INDEX " + indexName + " ON " + fullTableName
@@ -174,11 +179,6 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
conn.createStatement().execute("CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)" + tableDDLOptions);
query = "SELECT * FROM " + fullTableName;
rs = conn.createStatement().executeQuery(query);
@@ -284,12 +284,6 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
// make sure that the tables are empty, but reachable
conn.createStatement().execute("CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)" + tableDDLOptions);
query = "SELECT * FROM " + fullTableName;
@@ -405,12 +399,6 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
// make sure that the tables are empty, but reachable
conn.createStatement().execute(
"CREATE TABLE " + fullTableName
@@ -487,24 +475,20 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
@Test
public void testUpsertingNullForIndexedColumns() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String testTableName = tableName + "_" + System.currentTimeMillis();
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexeName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
Statement stmt = conn.createStatement();
- stmt.execute("CREATE TABLE " + fullTableName + "(v1 VARCHAR PRIMARY KEY, v2 DOUBLE, v3 VARCHAR) "+tableDDLOptions);
- stmt.execute("CREATE " + (localIndex ? "LOCAL" : "") + " INDEX " + indexName + " ON " + fullTableName + " (v2) INCLUDE(v3)");
+ stmt.execute("CREATE TABLE " + testTableName + "(v1 VARCHAR PRIMARY KEY, v2 DOUBLE, v3 VARCHAR) "+tableDDLOptions);
+ stmt.execute("CREATE " + (localIndex ? "LOCAL" : "") + " INDEX " + indexName + " ON " + testTableName + " (v2) INCLUDE(v3)");
//create a row with value null for indexed column v2
- stmt.executeUpdate("upsert into " + fullTableName + " values('cc1', null, 'abc')");
+ stmt.executeUpdate("upsert into " + testTableName + " values('cc1', null, 'abc')");
conn.commit();
//assert values in index table
- rs = stmt.executeQuery("select * from " + fullIndexeName);
+ rs = stmt.executeQuery("select * from " + fullIndexName);
assertTrue(rs.next());
assertEquals(0, Doubles.compare(0, rs.getDouble(1)));
assertTrue(rs.wasNull());
@@ -513,7 +497,7 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
//assert values in data table
- rs = stmt.executeQuery("select v1, v2, v3 from " + fullTableName);
+ rs = stmt.executeQuery("select v1, v2, v3 from " + testTableName);
assertTrue(rs.next());
assertEquals("cc1", rs.getString(1));
assertEquals(0, Doubles.compare(0, rs.getDouble(2)));
@@ -522,11 +506,11 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
//update the previously null value for indexed column v2 to a non-null value 1.23
- stmt.executeUpdate("upsert into " + fullTableName + " values('cc1', 1.23, 'abc')");
+ stmt.executeUpdate("upsert into " + testTableName + " values('cc1', 1.23, 'abc')");
conn.commit();
//assert values in data table
- rs = stmt.executeQuery("select /*+ NO_INDEX */ v1, v2, v3 from " + fullTableName);
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ v1, v2, v3 from " + testTableName);
assertTrue(rs.next());
assertEquals("cc1", rs.getString(1));
assertEquals(0, Doubles.compare(1.23, rs.getDouble(2)));
@@ -534,7 +518,7 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
//assert values in index table
- rs = stmt.executeQuery("select * from " + fullIndexeName);
+ rs = stmt.executeQuery("select * from " + indexName);
assertTrue(rs.next());
assertEquals(0, Doubles.compare(1.23, rs.getDouble(1)));
assertEquals("cc1", rs.getString(2));
@@ -542,11 +526,11 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
//update the value for indexed column v2 back to null
- stmt.executeUpdate("upsert into " + fullTableName + " values('cc1', null, 'abc')");
+ stmt.executeUpdate("upsert into " + testTableName + " values('cc1', null, 'abc')");
conn.commit();
//assert values in index table
- rs = stmt.executeQuery("select * from " + fullIndexeName);
+ rs = stmt.executeQuery("select * from " + indexName);
assertTrue(rs.next());
assertEquals(0, Doubles.compare(0, rs.getDouble(1)));
assertTrue(rs.wasNull());
@@ -555,7 +539,7 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
//assert values in data table
- rs = stmt.executeQuery("select v1, v2, v3 from " + fullTableName);
+ rs = stmt.executeQuery("select v1, v2, v3 from " + testTableName);
assertTrue(rs.next());
assertEquals("cc1", rs.getString(1));
assertEquals(0, Doubles.compare(0, rs.getDouble(2)));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java
new file mode 100644
index 0000000..58f6226
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java
@@ -0,0 +1,510 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end.index.txn;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.end2end.Shadower;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import com.google.common.collect.Maps;
+
+@RunWith(Parameterized.class)
+public class MutableRollbackIT extends BaseHBaseManagedTimeIT {
+
+ private final boolean localIndex;
+ private String tableName1;
+ private String indexName1;
+ private String fullTableName1;
+ private String tableName2;
+ private String indexName2;
+ private String fullTableName2;
+
+ public MutableRollbackIT(boolean localIndex) {
+ this.localIndex = localIndex;
+ this.tableName1 = TestUtil.DEFAULT_DATA_TABLE_NAME + "_1_";
+ this.indexName1 = "IDX1";
+ this.fullTableName1 = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName1);
+ this.tableName2 = TestUtil.DEFAULT_DATA_TABLE_NAME + "_2_";
+ this.indexName2 = "IDX2";
+ this.fullTableName2 = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName2);
+ }
+
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(2);
+ props.put(QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @Parameters(name="localIndex = {0}")
+ public static Collection<Boolean> data() {
+ return Arrays.asList(new Boolean[] {
+ false, true
+ });
+ }
+
+ public void testRollbackOfUncommittedExistingKeyValueIndexUpdate() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName1 + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
+ stmt.execute("CREATE TABLE " + fullTableName2 + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) IMMUTABLE_ROWS=true");
+ stmt.execute("CREATE "+(localIndex? " LOCAL " : "")+"INDEX " + indexName1 + " ON " + fullTableName1 + " (v1) INCLUDE(v2)");
+ stmt.execute("CREATE "+(localIndex? " LOCAL " : "")+"INDEX " + indexName2 + " ON " + fullTableName2 + " (v1) INCLUDE(v2)");
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'y', 'a')");
+ conn.commit();
+
+ //assert rows exists in fullTableName1
+ ResultSet rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert rows exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert no rows exists in fullTableName2
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ //assert no rows exists in indexName2
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName2 + ")*/ k, v1 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'y', 'b')");
+ stmt.executeUpdate("upsert into " + fullTableName2 + " values('a', 'b', 'c')");
+
+ //assert new covered column value
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("b", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert new covered column value
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("b", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert rows exists in fullTableName2
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName2);
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals("b", rs.getString(2));
+ assertEquals("c", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert rows exists in " + fullTableName2 + " index table
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName2 + ")*/ k, v1 from " + fullTableName2);
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals("b", rs.getString(2));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ //assert original row exists in fullTableName1
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert original row exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert no rows exists in fullTableName2
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ //assert no rows exists in indexName2
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName2 + ")*/ k, v1 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'z', 'a')");
+ stmt.executeUpdate("upsert into " + fullTableName2 + " values('a', 'b', 'c')");
+ conn.commit();
+
+ assertDataAndIndexRows(stmt);
+ stmt.executeUpdate("delete from " + fullTableName1 + " where k='x'");
+ stmt.executeUpdate("delete from " + fullTableName2 + " where v1='b'");
+
+ //assert no rows exists in fullTableName1
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName1);
+ assertFalse(rs.next());
+ //assert no rows exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertFalse(rs.next());
+
+ //assert no rows exists in fullTableName2
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName2);
+ assertFalse(rs.next());
+ //assert no rows exists in indexName2
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName2 + ")*/ k, v1 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ conn.rollback();
+ assertDataAndIndexRows(stmt);
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testRollbackOfUncommittedExistingRowKeyIndexUpdate() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName1 + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
+ stmt.execute("CREATE TABLE " + fullTableName2 + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) IMMUTABLE_ROWS=true");
+ stmt.execute("CREATE "+(localIndex? " LOCAL " : "")+"INDEX " + indexName1 + " ON " + fullTableName1 + " (v1, k)");
+ stmt.execute("CREATE "+(localIndex? " LOCAL " : "")+"INDEX " + indexName2 + " ON " + fullTableName2 + " (v1, k)");
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'y', 'a')");
+ conn.commit();
+
+ //assert rows exists in " + fullTableName1 + "
+ ResultSet rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert rows exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertFalse(rs.next());
+
+ //assert no rows exists in fullTableName2
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ //assert no rows exists in indexName2
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName2 + ")*/ k, v1 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'z', 'a')");
+ stmt.executeUpdate("upsert into " + fullTableName2 + " values('a', 'b', 'c')");
+
+ assertDataAndIndexRows(stmt);
+
+ conn.rollback();
+
+ //assert original row exists in fullTableName1
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert original row exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert no rows exists in fullTableName2
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ //assert no rows exists in indexName2
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'z', 'a')");
+ stmt.executeUpdate("upsert into " + fullTableName2 + " values('a', 'b', 'c')");
+ conn.commit();
+
+ assertDataAndIndexRows(stmt);
+ stmt.executeUpdate("delete from " + fullTableName1 + " where k='x'");
+ stmt.executeUpdate("delete from " + fullTableName2 + " where v1='b'");
+
+ //assert no rows exists in fullTableName1
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertFalse(rs.next());
+ //assert no rows exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1);
+ assertFalse(rs.next());
+
+ //assert no rows exists in fullTableName2
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName2);
+ assertFalse(rs.next());
+ //assert no rows exists in indexName2
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName2 + ")*/ k, v1 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ conn.rollback();
+ assertDataAndIndexRows(stmt);
+
+ } finally {
+ conn.close();
+ }
+ }
+
+ private void assertDataAndIndexRows(Statement stmt) throws SQLException, IOException {
+ ResultSet rs;
+ //assert new covered row key value exists in fullTableName1
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("z", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert new covered row key value exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("z", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert rows exists in fullTableName2
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName2);
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals("b", rs.getString(2));
+ assertEquals("c", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert rows exists in " + fullTableName2 + " index table
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName2);
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals("b", rs.getString(2));
+ assertFalse(rs.next());
+ }
+
+ @Test
+ public void testMultiRollbackOfUncommittedExistingRowKeyIndexUpdate() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName1 + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
+ stmt.execute("CREATE "+(localIndex? " LOCAL " : "")+"INDEX " + indexName1 + " ON " + fullTableName1 + " (v1, k)");
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'yyyy', 'a')");
+ conn.commit();
+
+ //assert rows exists in " + fullTableName1 + "
+ ResultSet rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("yyyy", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert rows exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("yyyy", rs.getString(2));
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'zzz', 'a')");
+
+ //assert new covered row key value exists in fullTableName1
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("zzz", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert new covered row key value exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("zzz", rs.getString(2));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ //assert original row exists in fullTableName1
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("yyyy", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert original row exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("yyyy", rs.getString(2));
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'zz', 'a')");
+
+ //assert new covered row key value exists in fullTableName1
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("zz", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert new covered row key value exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("zz", rs.getString(2));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ //assert original row exists in fullTableName1
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("yyyy", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert original row exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("yyyy", rs.getString(2));
+ assertFalse(rs.next());
+
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testCheckpointAndRollback() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName1 + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
+ stmt.execute("CREATE "+(localIndex? " LOCAL " : "")+"INDEX " + indexName1 + " ON " + fullTableName1 + " (v1)");
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'a', 'a')");
+ conn.commit();
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + "(k,v1) SELECT k,v1||'a' FROM " + fullTableName1);
+ ResultSet rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("aa", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("aa", rs.getString(2));
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + "(k,v1) SELECT k,v1||'a' FROM " + fullTableName1);
+
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("aaa", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("aaa", rs.getString(2));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ //assert original row exists in fullTableName1
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("a", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert original row exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("a", rs.getString(2));
+ assertFalse(rs.next());
+
+ } finally {
+ conn.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java
new file mode 100644
index 0000000..dbddcb1
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end.index.txn;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.end2end.Shadower;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import com.google.common.collect.Maps;
+
+@RunWith(Parameterized.class)
+public class RollbackIT extends BaseHBaseManagedTimeIT {
+
+ private final boolean localIndex;
+ private final boolean mutable;
+ private final String tableName;
+ private final String indexName;
+ private final String fullTableName;
+
+ public RollbackIT(boolean localIndex, boolean mutable) {
+ this.localIndex = localIndex;
+ this.mutable = mutable;
+ this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME;
+ this.indexName = "IDX";
+ this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ }
+
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(2);
+ props.put(QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @Parameters(name="localIndex = {0} , mutable = {1}")
+ public static Collection<Boolean[]> data() {
+ return Arrays.asList(new Boolean[][] {
+ { false, false }, { false, true },
+ { true, false }, { true, true }
+ });
+ }
+
+ @Test
+ public void testRollbackOfUncommittedKeyValueIndexInsert() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"+(!mutable? " IMMUTABLE_ROWS=true" : ""));
+ stmt.execute("CREATE "+(localIndex? "LOCAL " : "")+"INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE(v2)");
+
+ stmt.executeUpdate("upsert into " + fullTableName + " values('x', 'y', 'a')");
+
+ //assert values in data table
+ ResultSet rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert values in index table
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName + ")*/ k, v1, v2 from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ //assert values in data table
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName);
+ assertFalse(rs.next());
+
+ //assert values in index table
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName + ")*/ k, v1, v2 from " + fullTableName);
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testRollbackOfUncommittedRowKeyIndexInsert() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName + "(k VARCHAR, v1 VARCHAR, v2 VARCHAR, CONSTRAINT pk PRIMARY KEY (v1, v2))"+(!mutable? " IMMUTABLE_ROWS=true" : ""));
+ stmt.execute("CREATE "+(localIndex? "LOCAL " : "")+"INDEX " + indexName + " ON " + fullTableName + "(v1, k)");
+
+ stmt.executeUpdate("upsert into " + fullTableName + " values('x', 'y', 'a')");
+
+ ResultSet rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName);
+
+ //assert values in data table
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert values in index table
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName + ")*/ k, v1 from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ //assert values in data table
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName);
+ assertFalse(rs.next());
+
+ //assert values in index table
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName + ")*/ k, v1 from " + fullTableName);
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java
new file mode 100644
index 0000000..0e16d97
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end.index.txn;
+
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR;
+import static org.apache.phoenix.util.PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PARAM;
+import static org.apache.phoenix.util.TestUtil.LOCALHOST;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
+import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.hbase.index.Indexer;
+import org.apache.phoenix.jdbc.PhoenixTestDriver;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import com.google.common.collect.Maps;
+
+@RunWith(Parameterized.class)
+public class TxWriteFailureIT extends BaseTest {
+
+ private static PhoenixTestDriver driver;
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ private static final String SCHEMA_NAME = "S";
+ private static final String DATA_TABLE_NAME = "T";
+ private static final String INDEX_TABLE_NAME = "I";
+ private static final String DATA_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, DATA_TABLE_NAME);
+ private static final String INDEX_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, INDEX_TABLE_NAME);
+ private static final String ROW_TO_FAIL = "fail";
+
+ private final boolean localIndex;
+ private final boolean mutable;
+
+ public TxWriteFailureIT(boolean localIndex, boolean mutable) {
+ this.localIndex = localIndex;
+ this.mutable = mutable;
+ }
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+ setUpConfigForMiniCluster(conf);
+ conf.setClass("hbase.coprocessor.region.classes", FailingRegionObserver.class, RegionObserver.class);
+ conf.setBoolean("hbase.coprocessor.abortonerror", false);
+ conf.setBoolean(Indexer.CHECK_VERSION_CONF_KEY, false);
+ TEST_UTIL.startMiniCluster();
+ String clientPort = TEST_UTIL.getConfiguration().get(
+ QueryServices.ZOOKEEPER_PORT_ATTRIB);
+ url = JDBC_PROTOCOL + JDBC_PROTOCOL_SEPARATOR + LOCALHOST
+ + JDBC_PROTOCOL_SEPARATOR + clientPort
+ + JDBC_PROTOCOL_TERMINATOR + PHOENIX_TEST_DRIVER_URL_PARAM;
+
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+ // Must update config before starting server
+ props.put(QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ driver = initAndRegisterDriver(url, new ReadOnlyProps(props.entrySet().iterator()));
+ clusterInitialized = true;
+ setupTxManager();
+ }
+
+ @Parameters(name="localIndex = {0} , mutable = {1}")
+ public static Collection<Boolean[]> data() {
+ return Arrays.asList(new Boolean[][] {
+ { false, false }, { false, true }, { true, false }, { true, true }
+ });
+ }
+
+ @Test
+ public void testIndexTableWriteFailure() throws Exception {
+ helpTestWriteFailure(true);
+ }
+
+ @Test
+ public void testDataTableWriteFailure() throws Exception {
+ helpTestWriteFailure(false);
+ }
+
+ private void helpTestWriteFailure(boolean indexTableWriteFailure) throws SQLException {
+ ResultSet rs;
+
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = driver.connect(url, props);
+ conn.setAutoCommit(false);
+ conn.createStatement().execute(
+ "CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR PRIMARY KEY, v1 VARCHAR)"+(!mutable? " IMMUTABLE_ROWS=true" : ""));
+ conn.createStatement().execute(
+ "CREATE "+(localIndex? "LOCAL " : "")+"INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v1)");
+
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?)");
+ // to create a data table write failure set k as the ROW_TO_FAIL, to create an index table write failure set v1 as the ROW_TO_FAIL,
+ // FailingRegionObserver will throw an exception if the put contains ROW_TO_FAIL
+ stmt.setString(1, !indexTableWriteFailure ? ROW_TO_FAIL : "k1");
+ stmt.setString(2, indexTableWriteFailure ? ROW_TO_FAIL : "k2");
+ stmt.execute();
+ stmt.setString(1, "k2");
+ stmt.setString(2, "v2");
+ stmt.execute();
+ try {
+ conn.commit();
+ fail();
+ }
+ catch (Exception e) {
+ conn.rollback();
+ }
+ stmt.setString(1, "k3");
+ stmt.setString(2, "v3");
+ stmt.execute();
+ //this should pass
+ conn.commit();
+
+ // verify that only k3,v3 exists in the data table
+ String dataSql = "SELECT k, v1 FROM " + DATA_TABLE_FULL_NAME + " order by k";
+ rs = conn.createStatement().executeQuery("EXPLAIN "+dataSql);
+ assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER S.T",
+ QueryUtil.getExplainPlan(rs));
+ rs = conn.createStatement().executeQuery(dataSql);
+ assertTrue(rs.next());
+ assertEquals("k3", rs.getString(1));
+ assertEquals("v3", rs.getString(2));
+ assertFalse(rs.next());
+
+ // verify the only k3,v3 exists in the index table
+ String indexSql = "SELECT k, v1 FROM " + DATA_TABLE_FULL_NAME + " order by v1";
+ rs = conn.createStatement().executeQuery("EXPLAIN "+indexSql);
+ if(localIndex) {
+ assertEquals(
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _LOCAL_IDX_" + DATA_TABLE_FULL_NAME + " [-32768]\n" +
+ " SERVER FILTER BY FIRST KEY ONLY\n" +
+ "CLIENT MERGE SORT",
+ QueryUtil.getExplainPlan(rs));
+ } else {
+ assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME + "\n SERVER FILTER BY FIRST KEY ONLY",
+ QueryUtil.getExplainPlan(rs));
+ }
+ rs = conn.createStatement().executeQuery(indexSql);
+ assertTrue(rs.next());
+ assertEquals("k3", rs.getString(1));
+ assertEquals("v3", rs.getString(2));
+ assertFalse(rs.next());
+
+ conn.createStatement().execute("DROP TABLE " + DATA_TABLE_FULL_NAME);
+ }
+
+
+ public static class FailingRegionObserver extends SimpleRegionObserver {
+ @Override
+ public void prePut(ObserverContext<RegionCoprocessorEnvironment> c, Put put, WALEdit edit,
+ final Durability durability) throws HBaseIOException {
+ if (shouldFailUpsert(c, put)) {
+ // throwing anything other than instances of IOException result
+ // in this coprocessor being unloaded
+ // DoNotRetryIOException tells HBase not to retry this mutation
+ // multiple times
+ throw new DoNotRetryIOException();
+ }
+ }
+
+ private boolean shouldFailUpsert(ObserverContext<RegionCoprocessorEnvironment> c, Put put) {
+ return Bytes.contains(put.getRow(), Bytes.toBytes(ROW_TO_FAIL));
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java b/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
index e0f0a3c..4b06834 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
@@ -22,8 +22,6 @@ package org.apache.phoenix.execute;
import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Sets.newHashSet;
import static java.util.Collections.singletonList;
-import static org.apache.phoenix.query.BaseTest.initAndRegisterDriver;
-import static org.apache.phoenix.query.BaseTest.setUpConfigForMiniCluster;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR;
@@ -39,6 +37,8 @@ import java.sql.Driver;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
@@ -57,38 +58,37 @@ import org.apache.hadoop.hbase.coprocessor.RegionObserver;
import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.hbase.index.Indexer;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.TableRef;
-import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
-@Category(NeedsOwnMiniClusterTest.class)
-public class PartialCommitIT {
+@RunWith(Parameterized.class)
+public class PartialCommitIT extends BaseTest {
- private static final String TABLE_NAME_TO_FAIL = "b_failure_table".toUpperCase();
- private static final byte[] ROW_TO_FAIL = Bytes.toBytes("fail me");
- private static final String UPSERT_TO_FAIL = "upsert into " + TABLE_NAME_TO_FAIL + " values ('" + Bytes.toString(ROW_TO_FAIL) + "', 'boom!')";
- private static final String UPSERT_SELECT_TO_FAIL = "upsert into " + TABLE_NAME_TO_FAIL + " select k, c from a_success_table";
- private static final String DELETE_TO_FAIL = "delete from " + TABLE_NAME_TO_FAIL + " where k='z'";
+ private final String A_SUCESS_TABLE;
+ private final String B_FAILURE_TABLE;
+ private final String C_SUCESS_TABLE;
+ private final String UPSERT_TO_FAIL;
+ private final String UPSERT_SELECT_TO_FAIL;
+ private final String DELETE_TO_FAIL;
+ private static final String TABLE_NAME_TO_FAIL = "B_FAILURE_TABLE";
+ private static final byte[] ROW_TO_FAIL_UPSERT_BYTES = Bytes.toBytes("fail me upsert");
+ private static final byte[] ROW_TO_FAIL_DELETE_BYTES = Bytes.toBytes("fail me delete");
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- private static String url;
private static Driver driver;
- private static final Properties props = new Properties();
-
- static {
- props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, 10);
- }
@BeforeClass
public static void setupCluster() throws Exception {
@@ -104,29 +104,50 @@ public class PartialCommitIT {
Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
// Must update config before starting server
- props.put(QueryServices.DROP_METADATA_ATTRIB, Boolean.toString(true));
- driver = initAndRegisterDriver(url, new ReadOnlyProps(props.entrySet().iterator()));
+ driver = initAndRegisterDriver(url, ReadOnlyProps.EMPTY_PROPS);
+ clusterInitialized = true;
+ setupTxManager();
createTablesWithABitOfData();
}
+ @Parameters(name="transactional = {0}")
+ public static Collection<Boolean> data() {
+ return Arrays.asList(false, true);
+ }
+
+ public PartialCommitIT(boolean transactional) {
+ if (transactional) {
+ A_SUCESS_TABLE = "A_SUCCESS_TABLE_TXN";
+ B_FAILURE_TABLE = TABLE_NAME_TO_FAIL+"_TXN";
+ C_SUCESS_TABLE = "C_SUCCESS_TABLE_TXN";
+ }
+ else {
+ A_SUCESS_TABLE = "A_SUCCESS_TABLE";
+ B_FAILURE_TABLE = TABLE_NAME_TO_FAIL;
+ C_SUCESS_TABLE = "C_SUCCESS_TABLE";
+ }
+ UPSERT_TO_FAIL = "upsert into " + B_FAILURE_TABLE + " values ('" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "', 'boom!')";
+ UPSERT_SELECT_TO_FAIL = "upsert into " + B_FAILURE_TABLE + " select k, c from a_success_table";
+ DELETE_TO_FAIL = "delete from " + B_FAILURE_TABLE + " where k='" + Bytes.toString(ROW_TO_FAIL_DELETE_BYTES) + "'";
+ }
+
private static void createTablesWithABitOfData() throws Exception {
- Properties props = new Properties();
- props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, 10);
-
try (Connection con = driver.connect(url, new Properties())) {
Statement sta = con.createStatement();
sta.execute("create table a_success_table (k varchar primary key, c varchar)");
sta.execute("create table b_failure_table (k varchar primary key, c varchar)");
sta.execute("create table c_success_table (k varchar primary key, c varchar)");
+ sta.execute("create table a_success_table_txn (k varchar primary key, c varchar) TRANSACTIONAL=true");
+ sta.execute("create table b_failure_table_txn (k varchar primary key, c varchar) TRANSACTIONAL=true");
+ sta.execute("create table c_success_table_txn (k varchar primary key, c varchar) TRANSACTIONAL=true");
con.commit();
}
- props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, 100);
-
try (Connection con = driver.connect(url, new Properties())) {
con.setAutoCommit(false);
Statement sta = con.createStatement();
- for (String table : newHashSet("a_success_table", TABLE_NAME_TO_FAIL, "c_success_table")) {
+ for (String table : newHashSet("a_success_table", "b_failure_table", "c_success_table",
+ "a_success_table_txn", "b_failure_table_txn", "c_success_table_txn")) {
sta.execute("upsert into " + table + " values ('z', 'z')");
sta.execute("upsert into " + table + " values ('zz', 'zz')");
sta.execute("upsert into " + table + " values ('zzz', 'zzz')");
@@ -142,46 +163,44 @@ public class PartialCommitIT {
@Test
public void testNoFailure() {
- testPartialCommit(singletonList("upsert into a_success_table values ('testNoFailure', 'a')"), 0, new int[0], false,
- singletonList("select count(*) from a_success_table where k='testNoFailure'"), singletonList(new Integer(1)));
+ testPartialCommit(singletonList("upsert into " + A_SUCESS_TABLE + " values ('testNoFailure', 'a')"), 0, new int[0], false,
+ singletonList("select count(*) from " + A_SUCESS_TABLE + " where k='testNoFailure'"), singletonList(new Integer(1)));
}
@Test
public void testUpsertFailure() {
- testPartialCommit(newArrayList("upsert into a_success_table values ('testUpsertFailure1', 'a')",
+ testPartialCommit(newArrayList("upsert into " + A_SUCESS_TABLE + " values ('testUpsertFailure1', 'a')",
UPSERT_TO_FAIL,
- "upsert into a_success_table values ('testUpsertFailure2', 'b')"),
+ "upsert into " + A_SUCESS_TABLE + " values ('testUpsertFailure2', 'b')"),
1, new int[]{1}, true,
- newArrayList("select count(*) from a_success_table where k like 'testUpsertFailure_'",
- "select count(*) from " + TABLE_NAME_TO_FAIL + " where k = '" + Bytes.toString(ROW_TO_FAIL) + "'"),
+ newArrayList("select count(*) from " + A_SUCESS_TABLE + " where k like 'testUpsertFailure_'",
+ "select count(*) from " + B_FAILURE_TABLE + " where k = '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "'"),
newArrayList(new Integer(2), new Integer(0)));
}
@Test
public void testUpsertSelectFailure() throws SQLException {
- props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, 100);
-
try (Connection con = driver.connect(url, new Properties())) {
- con.createStatement().execute("upsert into a_success_table values ('" + Bytes.toString(ROW_TO_FAIL) + "', 'boom!')");
+ con.createStatement().execute("upsert into " + A_SUCESS_TABLE + " values ('" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "', 'boom!')");
con.commit();
}
- testPartialCommit(newArrayList("upsert into a_success_table values ('testUpsertSelectFailure', 'a')",
+ testPartialCommit(newArrayList("upsert into " + A_SUCESS_TABLE + " values ('testUpsertSelectFailure', 'a')",
UPSERT_SELECT_TO_FAIL),
1, new int[]{1}, true,
- newArrayList("select count(*) from a_success_table where k in ('testUpsertSelectFailure', '" + Bytes.toString(ROW_TO_FAIL) + "')",
- "select count(*) from " + TABLE_NAME_TO_FAIL + " where k = '" + Bytes.toString(ROW_TO_FAIL) + "'"),
+ newArrayList("select count(*) from " + A_SUCESS_TABLE + " where k in ('testUpsertSelectFailure', '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "')",
+ "select count(*) from " + B_FAILURE_TABLE + " where k = '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "'"),
newArrayList(new Integer(2), new Integer(0)));
}
@Test
public void testDeleteFailure() {
- testPartialCommit(newArrayList("upsert into a_success_table values ('testDeleteFailure1', 'a')",
+ testPartialCommit(newArrayList("upsert into " + A_SUCESS_TABLE + " values ('testDeleteFailure1', 'a')",
DELETE_TO_FAIL,
- "upsert into a_success_table values ('testDeleteFailure2', 'b')"),
+ "upsert into " + A_SUCESS_TABLE + " values ('testDeleteFailure2', 'b')"),
1, new int[]{1}, true,
- newArrayList("select count(*) from a_success_table where k like 'testDeleteFailure_'",
- "select count(*) from " + TABLE_NAME_TO_FAIL + " where k = 'z'"),
+ newArrayList("select count(*) from " + A_SUCESS_TABLE + " where k like 'testDeleteFailure_'",
+ "select count(*) from " + B_FAILURE_TABLE + " where k = 'z'"),
newArrayList(new Integer(2), new Integer(1)));
}
@@ -190,27 +209,27 @@ public class PartialCommitIT {
*/
@Test
public void testOrderOfMutationsIsPredicatable() {
- testPartialCommit(newArrayList("upsert into c_success_table values ('testOrderOfMutationsIsPredicatable', 'c')", // will fail because c_success_table is after b_failure_table by table sort order
+ testPartialCommit(newArrayList("upsert into " + C_SUCESS_TABLE + " values ('testOrderOfMutationsIsPredicatable', 'c')", // will fail because c_success_table is after b_failure_table by table sort order
UPSERT_TO_FAIL,
- "upsert into a_success_table values ('testOrderOfMutationsIsPredicatable', 'a')"), // will succeed because a_success_table is before b_failure_table by table sort order
+ "upsert into " + A_SUCESS_TABLE + " values ('testOrderOfMutationsIsPredicatable', 'a')"), // will succeed because a_success_table is before b_failure_table by table sort order
2, new int[]{0,1}, true,
- newArrayList("select count(*) from c_success_table where k='testOrderOfMutationsIsPredicatable'",
- "select count(*) from a_success_table where k='testOrderOfMutationsIsPredicatable'",
- "select count(*) from " + TABLE_NAME_TO_FAIL + " where k = '" + Bytes.toString(ROW_TO_FAIL) + "'"),
+ newArrayList("select count(*) from " + C_SUCESS_TABLE + " where k='testOrderOfMutationsIsPredicatable'",
+ "select count(*) from " + A_SUCESS_TABLE + " where k='testOrderOfMutationsIsPredicatable'",
+ "select count(*) from " + B_FAILURE_TABLE + " where k = '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "'"),
newArrayList(new Integer(0), new Integer(1), new Integer(0)));
}
@Test
- public void checkThatAllStatementTypesMaintainOrderInConnection() {
- testPartialCommit(newArrayList("upsert into a_success_table values ('k', 'checkThatAllStatementTypesMaintainOrderInConnection')",
- "upsert into a_success_table select k, c from c_success_table",
+ public void testStatementOrderMaintainedInConnection() {
+ testPartialCommit(newArrayList("upsert into " + A_SUCESS_TABLE + " values ('testStatementOrderMaintainedInConnection', 'a')",
+ "upsert into " + A_SUCESS_TABLE + " select k, c from " + C_SUCESS_TABLE,
DELETE_TO_FAIL,
- "select * from a_success_table",
+ "select * from " + A_SUCESS_TABLE + "",
UPSERT_TO_FAIL),
2, new int[]{2,4}, true,
- newArrayList("select count(*) from a_success_table where k='testOrderOfMutationsIsPredicatable' or k like 'z%'", // rows left: zz, zzz, checkThatAllStatementTypesMaintainOrderInConnection
- "select count(*) from " + TABLE_NAME_TO_FAIL + " where k = '" + ROW_TO_FAIL + "'",
- "select count(*) from " + TABLE_NAME_TO_FAIL + " where k = 'z'"),
+ newArrayList("select count(*) from " + A_SUCESS_TABLE + " where k='testStatementOrderMaintainedInConnection' or k like 'z%'", // rows left: zz, zzz, checkThatAllStatementTypesMaintainOrderInConnection
+ "select count(*) from " + B_FAILURE_TABLE + " where k = '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "'",
+ "select count(*) from " + B_FAILURE_TABLE + " where k = 'z'"),
newArrayList(new Integer(4), new Integer(0), new Integer(1)));
}
@@ -259,10 +278,11 @@ public class PartialCommitIT {
Connection con = driver.connect(url, new Properties());
PhoenixConnection phxCon = new PhoenixConnection(con.unwrap(PhoenixConnection.class));
final Map<TableRef,Map<ImmutableBytesPtr,MutationState.RowMutationState>> mutations = Maps.newTreeMap(new TableRefComparator());
- return new PhoenixConnection(phxCon) {
+ // passing a null mutation staate forces the connection.newMutationState() to be used to create the MutationState
+ return new PhoenixConnection(phxCon, null) {
@Override
protected MutationState newMutationState(int maxSize) {
- return new MutationState(maxSize, this, mutations);
+ return new MutationState(maxSize, this, mutations, null);
};
};
}
@@ -271,7 +291,7 @@ public class PartialCommitIT {
@Override
public void prePut(ObserverContext<RegionCoprocessorEnvironment> c, Put put, WALEdit edit,
final Durability durability) throws HBaseIOException {
- if (shouldFailUpsert(c, put)) {
+ if (shouldFail(c, put)) {
// throwing anything other than instances of IOException result
// in this coprocessor being unloaded
// DoNotRetryIOException tells HBase not to retry this mutation
@@ -283,7 +303,7 @@ public class PartialCommitIT {
@Override
public void preDelete(ObserverContext<RegionCoprocessorEnvironment> c,
Delete delete, WALEdit edit, Durability durability) throws IOException {
- if (shouldFailDelete(c, delete)) {
+ if (shouldFail(c, delete)) {
// throwing anything other than instances of IOException result
// in this coprocessor being unloaded
// DoNotRetryIOException tells HBase not to retry this mutation
@@ -292,18 +312,13 @@ public class PartialCommitIT {
}
}
- private boolean shouldFailUpsert(ObserverContext<RegionCoprocessorEnvironment> c, Put put) {
+ private boolean shouldFail(ObserverContext<RegionCoprocessorEnvironment> c, Mutation m) {
String tableName = c.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString();
- return TABLE_NAME_TO_FAIL.equals(tableName) && Bytes.equals(ROW_TO_FAIL, put.getRow());
+ // deletes on transactional tables are converted to put, so use a single helper method
+ return tableName.contains(TABLE_NAME_TO_FAIL) &&
+ (Bytes.equals(ROW_TO_FAIL_UPSERT_BYTES, m.getRow()) || Bytes.equals(ROW_TO_FAIL_DELETE_BYTES, m.getRow()));
}
- private boolean shouldFailDelete(ObserverContext<RegionCoprocessorEnvironment> c, Delete delete) {
- String tableName = c.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString();
- return TABLE_NAME_TO_FAIL.equals(tableName) &&
- // Phoenix deletes are sent as Mutations with empty values
- delete.getFamilyCellMap().firstEntry().getValue().get(0).getValueLength() == 0 &&
- delete.getFamilyCellMap().firstEntry().getValue().get(0).getQualifierLength() == 0;
- }
}
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
index b7f880a..98932e1 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
@@ -20,9 +20,8 @@ package org.apache.phoenix.rpc;
import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA;
import static org.apache.phoenix.util.TestUtil.MUTABLE_INDEX_DATA_TABLE;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-import static org.junit.Assert.assertEquals;
+import static org.apache.phoenix.util.TestUtil.TRANSACTIONAL_DATA_TABLE;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.eq;
import static org.mockito.Matchers.isNull;
@@ -30,33 +29,26 @@ import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
-import java.math.BigDecimal;
import java.sql.Connection;
-import java.sql.Date;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.Map;
import java.util.Properties;
import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
-import org.apache.phoenix.end2end.Shadower;
import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.MetaDataClient;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.types.PVarchar;
-import org.apache.phoenix.util.DateUtil;
+import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
-import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.TestUtil;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
-import com.google.common.collect.Maps;
-
/**
* Verifies the number of rpcs calls from {@link MetaDataClient} updateCache()
* for transactional and non-transactional tables.
@@ -68,86 +60,76 @@ public class UpdateCacheIT extends BaseHBaseManagedTimeIT {
@Before
public void setUp() throws SQLException {
ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE);
+ ensureTableCreated(getUrl(), TRANSACTIONAL_DATA_TABLE);
}
- @BeforeClass
- @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
- public static void doSetup() throws Exception {
- Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
- setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ @Test
+ public void testUpdateCacheForTxnTable() throws Exception {
+ helpTestUpdateCache(true, null);
}
-
- public static void validateRowKeyColumns(ResultSet rs, int i) throws SQLException {
- assertTrue(rs.next());
- assertEquals(rs.getString(1), "varchar" + String.valueOf(i));
- assertEquals(rs.getString(2), "char" + String.valueOf(i));
- assertEquals(rs.getInt(3), i);
- assertEquals(rs.getInt(4), i);
- assertEquals(rs.getBigDecimal(5), new BigDecimal(i*0.5d));
- Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * NUM_MILLIS_IN_DAY);
- assertEquals(rs.getDate(6), date);
- }
-
- public static void setRowKeyColumns(PreparedStatement stmt, int i) throws SQLException {
- // insert row
- stmt.setString(1, "varchar" + String.valueOf(i));
- stmt.setString(2, "char" + String.valueOf(i));
- stmt.setInt(3, i);
- stmt.setLong(4, i);
- stmt.setBigDecimal(5, new BigDecimal(i*0.5d));
- Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * NUM_MILLIS_IN_DAY);
- stmt.setDate(6, date);
+
+ @Test
+ public void testUpdateCacheForNonTxnTable() throws Exception {
+ helpTestUpdateCache(false, null);
}
- @Test
- public void testUpdateCache() throws Exception {
- String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE;
+ public static void helpTestUpdateCache(boolean isTransactional, Long scn) throws Exception {
+ String tableName = isTransactional ? TRANSACTIONAL_DATA_TABLE : MUTABLE_INDEX_DATA_TABLE;
+ String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + tableName;
String selectSql = "SELECT * FROM "+fullTableName;
// use a spyed ConnectionQueryServices so we can verify calls to getTable
ConnectionQueryServices connectionQueryServices = Mockito.spy(driver.getConnectionQueryServices(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)));
Properties props = new Properties();
props.putAll(PhoenixEmbeddedDriver.DEFFAULT_PROPS.asMap());
+ if (scn!=null) {
+ props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(scn));
+ }
Connection conn = connectionQueryServices.connect(getUrl(), props);
try {
conn.setAutoCommit(false);
- ResultSet rs = conn.createStatement().executeQuery(selectSql);
- assertFalse(rs.next());
- reset(connectionQueryServices);
-
String upsert = "UPSERT INTO " + fullTableName + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
PreparedStatement stmt = conn.prepareStatement(upsert);
// upsert three rows
- setRowKeyColumns(stmt, 1);
+ TestUtil.setRowKeyColumns(stmt, 1);
stmt.execute();
- setRowKeyColumns(stmt, 2);
+ TestUtil.setRowKeyColumns(stmt, 2);
stmt.execute();
- setRowKeyColumns(stmt, 3);
+ TestUtil.setRowKeyColumns(stmt, 3);
stmt.execute();
conn.commit();
- // verify only one rpc to getTable occurs after commit is called
- verify(connectionQueryServices, times(1)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(INDEX_DATA_SCHEMA)), eq(PVarchar.INSTANCE.toBytes(MUTABLE_INDEX_DATA_TABLE)), anyLong(), anyLong());
- reset(connectionQueryServices);
+ // verify only one rpc to fetch table metadata,
+ verify(connectionQueryServices).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(INDEX_DATA_SCHEMA)), eq(PVarchar.INSTANCE.toBytes(tableName)), anyLong(), anyLong());
+ reset(connectionQueryServices);
+
+ if (scn!=null) {
+ // advance scn so that we can see the data we just upserted
+ props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(scn+2));
+ conn = connectionQueryServices.connect(getUrl(), props);
+ }
- rs = conn.createStatement().executeQuery(selectSql);
- validateRowKeyColumns(rs, 1);
- validateRowKeyColumns(rs, 2);
- validateRowKeyColumns(rs, 3);
+ ResultSet rs = conn.createStatement().executeQuery(selectSql);
+ TestUtil.validateRowKeyColumns(rs, 1);
+ TestUtil.validateRowKeyColumns(rs, 2);
+ TestUtil.validateRowKeyColumns(rs, 3);
assertFalse(rs.next());
rs = conn.createStatement().executeQuery(selectSql);
- validateRowKeyColumns(rs, 1);
- validateRowKeyColumns(rs, 2);
- validateRowKeyColumns(rs, 3);
+ TestUtil.validateRowKeyColumns(rs, 1);
+ TestUtil.validateRowKeyColumns(rs, 2);
+ TestUtil.validateRowKeyColumns(rs, 3);
assertFalse(rs.next());
rs = conn.createStatement().executeQuery(selectSql);
- validateRowKeyColumns(rs, 1);
- validateRowKeyColumns(rs, 2);
- validateRowKeyColumns(rs, 3);
+ TestUtil.validateRowKeyColumns(rs, 1);
+ TestUtil.validateRowKeyColumns(rs, 2);
+ TestUtil.validateRowKeyColumns(rs, 3);
assertFalse(rs.next());
- conn.commit();
- // there should be one rpc to getTable per query
- verify(connectionQueryServices, times(3)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(INDEX_DATA_SCHEMA)), eq(PVarchar.INSTANCE.toBytes(MUTABLE_INDEX_DATA_TABLE)), anyLong(), anyLong());
+
+ // for non-transactional tables without a scn : verify one rpc to getTable occurs *per* query
+ // for non-transactional tables with a scn : verify *only* one rpc occurs
+ // for transactional tables : verify *only* one rpc occurs
+ int numRpcs = isTransactional || scn!=null ? 1 : 3;
+ verify(connectionQueryServices, times(numRpcs)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(INDEX_DATA_SCHEMA)), eq(PVarchar.INSTANCE.toBytes(tableName)), anyLong(), anyLong());
}
finally {
conn.close();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java
new file mode 100644
index 0000000..dbc7fd1
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.rpc;
+
+import static org.apache.phoenix.util.TestUtil.MUTABLE_INDEX_DATA_TABLE;
+
+import org.apache.phoenix.end2end.BaseClientManagedTimeIT;
+import org.junit.Before;
+import org.junit.Test;
+
+public class UpdateCacheWithScnIT extends BaseClientManagedTimeIT {
+
+ protected long ts;
+
+ @Before
+ public void initTable() throws Exception {
+ ts = nextTimestamp();
+ ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE, ts);
+ }
+
+ @Test
+ public void testUpdateCacheWithScn() throws Exception {
+ UpdateCacheIT.helpTestUpdateCache(false, ts+2);
+ }
+
+}
[3/9] phoenix git commit: PHOENIX-1674 Snapshot isolation transaction
support through Tephra (James Taylor, Thomas D'Silva)
Posted by td...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 18a97bd..2a08f45 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -27,6 +27,7 @@ import static org.apache.phoenix.util.UpgradeUtil.upgradeTo4_5_0;
import java.io.IOException;
import java.sql.SQLException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -115,6 +116,7 @@ import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.hbase.index.util.VersionUtil;
import org.apache.phoenix.index.PhoenixIndexBuilder;
import org.apache.phoenix.index.PhoenixIndexCodec;
+import org.apache.phoenix.index.PhoenixTransactionalIndexer;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
@@ -163,9 +165,20 @@ import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.ServerUtil;
import org.apache.phoenix.util.UpgradeUtil;
+import org.apache.twill.discovery.ZKDiscoveryService;
+import org.apache.twill.zookeeper.RetryStrategies;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.apache.twill.zookeeper.ZKClientServices;
+import org.apache.twill.zookeeper.ZKClients;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import co.cask.tephra.TransactionSystemClient;
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.distributed.PooledClientProvider;
+import co.cask.tephra.distributed.TransactionServiceClient;
+import co.cask.tephra.hbase11.coprocessor.TransactionProcessor;
+
import com.google.common.base.Joiner;
import com.google.common.base.Throwables;
import com.google.common.cache.Cache;
@@ -183,6 +196,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
// Max number of cached table stats for view or shared index physical tables
private static final int MAX_TABLE_STATS_CACHE_ENTRIES = 512;
protected final Configuration config;
+ private final ConnectionInfo connectionInfo;
// Copy of config.getProps(), but read-only to prevent synchronization that we
// don't need.
private final ReadOnlyProps props;
@@ -197,7 +211,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
// Lowest HBase version on the cluster.
private int lowestClusterHBaseVersion = Integer.MAX_VALUE;
- private boolean hasInvalidIndexConfiguration = false;
+ private boolean isMutableIndexWALCodecInstalled = true;
@GuardedBy("connectionCountLock")
private int connectionCount = 0;
@@ -205,6 +219,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private final boolean returnSequenceValues ;
private HConnection connection;
+ private TransactionServiceClient txServiceClient;
private volatile boolean initialized;
private volatile int nSequenceSaltBuckets;
@@ -257,6 +272,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
for (Entry<String,String> entry : connectionInfo.asProps()) {
config.set(entry.getKey(), entry.getValue());
}
+ this.connectionInfo = connectionInfo;
// Without making a copy of the configuration we cons up, we lose some of our properties
// on the server side during testing.
@@ -282,6 +298,30 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
this.returnSequenceValues = config.getBoolean(QueryServices.RETURN_SEQUENCE_VALUES_ATTRIB, QueryServicesOptions.DEFAULT_RETURN_SEQUENCE_VALUES);
}
+ @Override
+ public TransactionSystemClient getTransactionSystemClient() {
+ return txServiceClient;
+ }
+
+ private void initTxServiceClient() {
+ String zkQuorumServersString = connectionInfo.getZookeeperQuorum()+":"+connectionInfo.getPort();
+ ZKClientService zkClientService = ZKClientServices.delegate(
+ ZKClients.reWatchOnExpire(
+ ZKClients.retryOnFailure(
+ ZKClientService.Builder.of(zkQuorumServersString)
+ .setSessionTimeout(config.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT))
+ .build(),
+ RetryStrategies.exponentialDelay(500, 2000, TimeUnit.MILLISECONDS)
+ )
+ )
+ );
+ zkClientService.startAndWait();
+ ZKDiscoveryService zkDiscoveryService = new ZKDiscoveryService(zkClientService);
+ PooledClientProvider pooledClientProvider = new PooledClientProvider(
+ config, zkDiscoveryService);
+ this.txServiceClient = new TransactionServiceClient(config,pooledClientProvider);
+ }
+
private void openConnection() throws SQLException {
try {
// check if we need to authenticate with kerberos
@@ -293,6 +333,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
User.login(config, HBASE_CLIENT_KEYTAB, HBASE_CLIENT_PRINCIPAL, null);
logger.info("Successfull login to secure cluster!!");
}
+ initTxServiceClient();
this.connection = HBaseFactoryProvider.getHConnectionFactory().createConnection(this.config);
} catch (IOException e) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ESTABLISH_CONNECTION)
@@ -308,9 +349,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
try {
return HBaseFactoryProvider.getHTableFactory().getTable(tableName, connection, null);
} catch (org.apache.hadoop.hbase.TableNotFoundException e) {
- byte[][] schemaAndTableName = new byte[2][];
- SchemaUtil.getVarChars(tableName, schemaAndTableName);
- throw new TableNotFoundException(Bytes.toString(schemaAndTableName[0]), Bytes.toString(schemaAndTableName[1]));
+ throw new TableNotFoundException(SchemaUtil.getSchemaNameFromFullName(tableName), SchemaUtil.getTableNameFromFullName(tableName));
} catch (IOException e) {
throw new SQLException(e);
}
@@ -457,18 +496,27 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
@Override
- public PMetaData addTable(PTable table) throws SQLException {
+ public PMetaData addTable(PTable table, long resolvedTime) throws SQLException {
synchronized (latestMetaDataLock) {
try {
throwConnectionClosedIfNullMetaData();
// If existing table isn't older than new table, don't replace
// If a client opens a connection at an earlier timestamp, this can happen
- PTable existingTable = latestMetaData.getTable(new PTableKey(table.getTenantId(), table.getName().getString()));
+ PTable existingTable = latestMetaData.getTableRef(new PTableKey(table.getTenantId(), table.getName().getString())).getTable();
if (existingTable.getTimeStamp() >= table.getTimeStamp()) {
return latestMetaData;
}
} catch (TableNotFoundException e) {}
- latestMetaData = latestMetaData.addTable(table);
+ latestMetaData = latestMetaData.addTable(table, resolvedTime);
+ latestMetaDataLock.notifyAll();
+ return latestMetaData;
+ }
+ }
+
+ public PMetaData updateResolvedTimestamp(PTable table, long resolvedTime) throws SQLException {
+ synchronized (latestMetaDataLock) {
+ throwConnectionClosedIfNullMetaData();
+ latestMetaData = latestMetaData.updateResolvedTimestamp(table, resolvedTime);
latestMetaDataLock.notifyAll();
return latestMetaData;
}
@@ -490,7 +538,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
while (true) {
try {
try {
- table = metaData.getTable(new PTableKey(tenantId, tableName));
+ table = metaData.getTableRef(new PTableKey(tenantId, tableName)).getTable();
/* If the table is at the prior sequence number, then we're good to go.
* We know if we've got this far, that the server validated the mutations,
* so we'd just need to wait until the other connection that mutated the same
@@ -531,12 +579,12 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
@Override
- public PMetaData addColumn(final PName tenantId, final String tableName, final List<PColumn> columns, final long tableTimeStamp, final long tableSeqNum, final boolean isImmutableRows, final boolean isWalDisabled, final boolean isMultitenant, final boolean storeNulls) throws SQLException {
+ public PMetaData addColumn(final PName tenantId, final String tableName, final List<PColumn> columns, final long tableTimeStamp, final long tableSeqNum, final boolean isImmutableRows, final boolean isWalDisabled, final boolean isMultitenant, final boolean storeNulls, final boolean isTransactional, final long resolvedTime) throws SQLException {
return metaDataMutated(tenantId, tableName, tableSeqNum, new Mutator() {
@Override
public PMetaData mutate(PMetaData metaData) throws SQLException {
try {
- return metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls);
+ return metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime);
} catch (TableNotFoundException e) {
// The DROP TABLE may have been processed first, so just ignore.
return metaData;
@@ -556,12 +604,12 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
@Override
- public PMetaData removeColumn(final PName tenantId, final String tableName, final List<PColumn> columnsToRemove, final long tableTimeStamp, final long tableSeqNum) throws SQLException {
+ public PMetaData removeColumn(final PName tenantId, final String tableName, final List<PColumn> columnsToRemove, final long tableTimeStamp, final long tableSeqNum, final long resolvedTime) throws SQLException {
return metaDataMutated(tenantId, tableName, tableSeqNum, new Mutator() {
@Override
public PMetaData mutate(PMetaData metaData) throws SQLException {
try {
- return metaData.removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum);
+ return metaData.removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum, resolvedTime);
} catch (TableNotFoundException e) {
// The DROP TABLE may have been processed first, so just ignore.
return metaData;
@@ -594,25 +642,40 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
for (Entry<String,Object> entry : family.getSecond().entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
- columnDesc.setValue(key, value == null ? null : value.toString());
+ setHColumnDescriptorValue(columnDesc, key, value);
}
}
return columnDesc;
}
-
- private void modifyColumnFamilyDescriptor(HColumnDescriptor hcd, Pair<byte[], Map<String,Object>> family) throws SQLException {
- if (Bytes.equals(hcd.getName(), family.getFirst())) {
- modifyColumnFamilyDescriptor(hcd, family.getSecond());
+
+ // Workaround HBASE-14737
+ private static void setHColumnDescriptorValue(HColumnDescriptor columnDesc, String key, Object value) {
+ if (HConstants.VERSIONS.equals(key)) {
+ columnDesc.setMaxVersions(getMaxVersion(value));
} else {
- throw new IllegalArgumentException("Column family names don't match. Column descriptor family name: " + hcd.getNameAsString() + ", Family name: " + Bytes.toString(family.getFirst()));
+ columnDesc.setValue(key, value == null ? null : value.toString());
}
}
+
+ private static int getMaxVersion(Object value) {
+ if (value == null) {
+ return -1; // HColumnDescriptor.UNINITIALIZED is private
+ }
+ if (value instanceof Number) {
+ return ((Number)value).intValue();
+ }
+ String stringValue = value.toString();
+ if (stringValue.isEmpty()) {
+ return -1;
+ }
+ return Integer.parseInt(stringValue);
+ }
private void modifyColumnFamilyDescriptor(HColumnDescriptor hcd, Map<String,Object> props) throws SQLException {
for (Entry<String, Object> entry : props.entrySet()) {
String propName = entry.getKey();
Object value = entry.getValue();
- hcd.setValue(propName, value == null ? null : value.toString());
+ setHColumnDescriptorValue(hcd, propName, value);
}
}
@@ -622,8 +685,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
new HTableDescriptor(TableName.valueOf(tableName));
for (Entry<String,Object> entry : tableProps.entrySet()) {
String key = entry.getKey();
- Object value = entry.getValue();
- tableDescriptor.setValue(key, value == null ? null : value.toString());
+ if (!TableProperty.isPhoenixTableProperty(key)) {
+ Object value = entry.getValue();
+ tableDescriptor.setValue(key, value == null ? null : value.toString());
+ }
}
if (families.isEmpty()) {
if (tableType != PTableType.VIEW) {
@@ -649,16 +714,20 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
tableDescriptor.addFamily(columnDescriptor);
} else {
if (tableType != PTableType.VIEW) {
- modifyColumnFamilyDescriptor(tableDescriptor.getFamily(familyByte), family);
+ HColumnDescriptor columnDescriptor = tableDescriptor.getFamily(familyByte);
+ if (columnDescriptor == null) {
+ throw new IllegalArgumentException("Unable to find column descriptor with family name " + Bytes.toString(family.getFirst()));
+ }
+ modifyColumnFamilyDescriptor(columnDescriptor, family.getSecond());
}
}
}
}
- addCoprocessors(tableName, tableDescriptor, tableType);
+ addCoprocessors(tableName, tableDescriptor, tableType, tableProps);
return tableDescriptor;
}
- private void addCoprocessors(byte[] tableName, HTableDescriptor descriptor, PTableType tableType) throws SQLException {
+ private void addCoprocessors(byte[] tableName, HTableDescriptor descriptor, PTableType tableType, Map<String,Object> tableProps) throws SQLException {
// The phoenix jar must be available on HBase classpath
int priority = props.getInt(QueryServices.COPROCESSOR_PRIORITY_ATTRIB, QueryServicesOptions.DEFAULT_COPROCESSOR_PRIORITY);
try {
@@ -674,17 +743,35 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (!descriptor.hasCoprocessor(ServerCachingEndpointImpl.class.getName())) {
descriptor.addCoprocessor(ServerCachingEndpointImpl.class.getName(), null, priority, null);
}
+ boolean isTransactional =
+ Boolean.TRUE.equals(tableProps.get(TableProperty.TRANSACTIONAL.name())) ||
+ Boolean.TRUE.equals(tableProps.get(TxConstants.READ_NON_TX_DATA)); // For ALTER TABLE
// TODO: better encapsulation for this
// Since indexes can't have indexes, don't install our indexing coprocessor for indexes.
// Also don't install on the SYSTEM.CATALOG and SYSTEM.STATS table because we use
// all-or-none mutate class which break when this coprocessor is installed (PHOENIX-1318).
if ((tableType != PTableType.INDEX && tableType != PTableType.VIEW)
&& !SchemaUtil.isMetaTable(tableName)
- && !SchemaUtil.isStatsTable(tableName)
- && !descriptor.hasCoprocessor(Indexer.class.getName())) {
- Map<String, String> opts = Maps.newHashMapWithExpectedSize(1);
- opts.put(NonTxIndexBuilder.CODEC_CLASS_NAME_KEY, PhoenixIndexCodec.class.getName());
- Indexer.enableIndexing(descriptor, PhoenixIndexBuilder.class, opts, priority);
+ && !SchemaUtil.isStatsTable(tableName)) {
+ if (isTransactional) {
+ if (!descriptor.hasCoprocessor(PhoenixTransactionalIndexer.class.getName())) {
+ descriptor.addCoprocessor(PhoenixTransactionalIndexer.class.getName(), null, priority, null);
+ }
+ // For alter table, remove non transactional index coprocessor
+ if (descriptor.hasCoprocessor(Indexer.class.getName())) {
+ descriptor.removeCoprocessor(Indexer.class.getName());
+ }
+ } else {
+ if (!descriptor.hasCoprocessor(Indexer.class.getName())) {
+ // If exception on alter table to transition back to non transactional
+ if (descriptor.hasCoprocessor(PhoenixTransactionalIndexer.class.getName())) {
+ descriptor.removeCoprocessor(PhoenixTransactionalIndexer.class.getName());
+ }
+ Map<String, String> opts = Maps.newHashMapWithExpectedSize(1);
+ opts.put(NonTxIndexBuilder.CODEC_CLASS_NAME_KEY, PhoenixIndexCodec.class.getName());
+ Indexer.enableIndexing(descriptor, PhoenixIndexBuilder.class, opts, priority);
+ }
+ }
}
if (SchemaUtil.isStatsTable(tableName) && !descriptor.hasCoprocessor(MultiRowMutationEndpoint.class.getName())) {
descriptor.addCoprocessor(MultiRowMutationEndpoint.class.getName(),
@@ -722,6 +809,17 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
descriptor.addCoprocessor(SequenceRegionObserver.class.getName(), null, priority, null);
}
}
+
+ if (isTransactional) {
+ if (!descriptor.hasCoprocessor(TransactionProcessor.class.getName())) {
+ descriptor.addCoprocessor(TransactionProcessor.class.getName(), null, priority - 10, null);
+ }
+ } else {
+ // If exception on alter table to transition back to non transactional
+ if (descriptor.hasCoprocessor(TransactionProcessor.class.getName())) {
+ descriptor.removeCoprocessor(TransactionProcessor.class.getName());
+ }
+ }
} catch (IOException e) {
throw ServerUtil.parseServerException(e);
}
@@ -879,8 +977,26 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
checkClientServerCompatibility();
}
- if (!modifyExistingMetaData || existingDesc.equals(newDesc)) {
- return existingDesc;
+ if (!modifyExistingMetaData) {
+ return existingDesc; // Caller already knows that no metadata was changed
+ }
+ boolean willBeTx = Boolean.TRUE.equals(props.get(TableProperty.TRANSACTIONAL.name()));
+ // If mapping an existing table as transactional, set property so that existing
+ // data is correctly read.
+ if (willBeTx) {
+ newDesc.setValue(TxConstants.READ_NON_TX_DATA, Boolean.TRUE.toString());
+ } else {
+ // If we think we're creating a non transactional table when it's already
+ // transactional, don't allow.
+ if (existingDesc.hasCoprocessor(TransactionProcessor.class.getName())) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.TX_MAY_NOT_SWITCH_TO_NON_TX)
+ .setSchemaName(SchemaUtil.getSchemaNameFromFullName(tableName))
+ .setTableName(SchemaUtil.getTableNameFromFullName(tableName)).build().buildException();
+ }
+ newDesc.remove(TxConstants.READ_NON_TX_DATA);
+ }
+ if (existingDesc.equals(newDesc)) {
+ return null; // Indicate that no metadata was changed
}
modifyTable(tableName, newDesc, true);
@@ -987,7 +1103,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
buf.append(name);
buf.append(';');
}
- hasInvalidIndexConfiguration |= isInvalidMutableIndexConfig(result.getValue());
+ isMutableIndexWALCodecInstalled &= !isInvalidMutableIndexConfig(result.getValue());
if (minHBaseVersion > MetaDataUtil.decodeHBaseVersion(result.getValue())) {
minHBaseVersion = MetaDataUtil.decodeHBaseVersion(result.getValue());
}
@@ -1096,7 +1212,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
try {
synchronized (latestMetaDataLock) {
throwConnectionClosedIfNullMetaData();
- table = latestMetaData.getTable(new PTableKey(PName.EMPTY_NAME, parentTableName));
+ table = latestMetaData.getTableRef(new PTableKey(PName.EMPTY_NAME, parentTableName)).getTable();
latestMetaDataLock.notifyAll();
}
if (table.getTimeStamp() >= timestamp) { // Table in cache is newer than client timestamp which shouldn't be the case
@@ -1271,7 +1387,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
builder.addTableMetadataMutations(mp.toByteString());
}
builder.setClientVersion(VersionUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER));
- instance.createTable(controller, builder.build(), rpcCallback);
+ CreateTableRequest build = builder.build();
+ instance.createTable(controller, build, rpcCallback);
if(controller.getFailedOn() != null) {
throw controller.getFailedOn();
}
@@ -1460,7 +1577,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (metadata == null) {
throwConnectionClosedException();
}
- table = metadata.getTable(new PTableKey(tenantId, name));
+ table = metadata.getTableRef(new PTableKey(tenantId, name)).getTable();
if (table.getTimeStamp() >= timestamp) { // Table in cache is newer than client timestamp which shouldn't be the case
throw new TableNotFoundException(table.getSchemaName().getString(), table.getTableName().getString());
}
@@ -1505,22 +1622,213 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
@Override
public MetaDataMutationResult addColumn(final List<Mutation> tableMetaData, PTable table, Map<String, List<Pair<String,Object>>> stmtProperties, Set<String> colFamiliesForPColumnsToBeAdded) throws SQLException {
- Map<String, Object> tableProps = new HashMap<String, Object>();
- HTableDescriptor tableDescriptor = separateAndValidateProperties(table, stmtProperties, colFamiliesForPColumnsToBeAdded, tableProps);
- SQLException sqlE = null;
+ List<Pair<byte[], Map<String, Object>>> families = new ArrayList<>(stmtProperties.size());
+ Map<String, Object> tableProps = new HashMap<String, Object>();
+ Set<HTableDescriptor> tableDescriptors = Collections.emptySet();
+ Set<HTableDescriptor> origTableDescriptors = Collections.emptySet();
+ boolean nonTxToTx = false;
+ Pair<HTableDescriptor,HTableDescriptor> tableDescriptorPair = separateAndValidateProperties(table, stmtProperties, colFamiliesForPColumnsToBeAdded, families, tableProps);
+ HTableDescriptor tableDescriptor = tableDescriptorPair.getSecond();
+ HTableDescriptor origTableDescriptor = tableDescriptorPair.getFirst();
if (tableDescriptor != null) {
- try {
- boolean modifyHTable = true;
- if (table.getType() == PTableType.VIEW) {
- boolean canViewsAddNewCF = props.getBoolean(QueryServices.ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE,
- QueryServicesOptions.DEFAULT_ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE);
- // When adding a column to a view, base physical table should only be modified when new column families are being added.
- modifyHTable = canViewsAddNewCF && !existingColumnFamiliesForBaseTable(table.getPhysicalName()).containsAll(colFamiliesForPColumnsToBeAdded);
+ tableDescriptors = Sets.newHashSetWithExpectedSize(3 + table.getIndexes().size());
+ origTableDescriptors = Sets.newHashSetWithExpectedSize(3 + table.getIndexes().size());
+ tableDescriptors.add(tableDescriptor);
+ origTableDescriptors.add(origTableDescriptor);
+ nonTxToTx = Boolean.TRUE.equals(tableProps.get(TxConstants.READ_NON_TX_DATA));
+ /*
+ * If the table was transitioned from non transactional to transactional, we need
+ * to also transition the index tables.
+ */
+ if (nonTxToTx) {
+ updateDescriptorForTx(table, tableProps, tableDescriptor, Boolean.TRUE.toString(), tableDescriptors, origTableDescriptors);
+ }
+ }
+
+ boolean success = false;
+ boolean metaDataUpdated = !tableDescriptors.isEmpty();
+ boolean pollingNeeded = !(!tableProps.isEmpty() && families.isEmpty() && colFamiliesForPColumnsToBeAdded.isEmpty());
+ MetaDataMutationResult result = null;
+ try {
+ boolean modifyHTable = true;
+ if (table.getType() == PTableType.VIEW) {
+ boolean canViewsAddNewCF = props.getBoolean(QueryServices.ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE,
+ QueryServicesOptions.DEFAULT_ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE);
+ // When adding a column to a view, base physical table should only be modified when new column families are being added.
+ modifyHTable = canViewsAddNewCF && !existingColumnFamiliesForBaseTable(table.getPhysicalName()).containsAll(colFamiliesForPColumnsToBeAdded);
+ }
+ if (modifyHTable) {
+ sendHBaseMetaData(tableDescriptors, pollingNeeded);
+ }
+
+ // Special case for call during drop table to ensure that the empty column family exists.
+ // In this, case we only include the table header row, as until we add schemaBytes and tableBytes
+ // as args to this function, we have no way of getting them in this case.
+ // TODO: change to if (tableMetaData.isEmpty()) once we pass through schemaBytes and tableBytes
+ // Also, could be used to update property values on ALTER TABLE t SET prop=xxx
+ if ((tableMetaData.isEmpty()) || (tableMetaData.size() == 1 && tableMetaData.get(0).isEmpty())) {
+ return new MetaDataMutationResult(MutationCode.NO_OP, System.currentTimeMillis(), table);
+ }
+ byte[][] rowKeyMetaData = new byte[3][];
+ PTableType tableType = table.getType();
+
+ Mutation m = tableMetaData.get(0);
+ byte[] rowKey = m.getRow();
+ SchemaUtil.getVarChars(rowKey, rowKeyMetaData);
+ byte[] tenantIdBytes = rowKeyMetaData[PhoenixDatabaseMetaData.TENANT_ID_INDEX];
+ byte[] schemaBytes = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
+ byte[] tableBytes = rowKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
+ byte[] tableKey = SchemaUtil.getTableKey(tenantIdBytes, schemaBytes, tableBytes);
+
+ ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+ result = metaDataCoprocessorExec(tableKey,
+ new Batch.Call<MetaDataService, MetaDataResponse>() {
+ @Override
+ public MetaDataResponse call(MetaDataService instance) throws IOException {
+ ServerRpcController controller = new ServerRpcController();
+ BlockingRpcCallback<MetaDataResponse> rpcCallback =
+ new BlockingRpcCallback<MetaDataResponse>();
+ AddColumnRequest.Builder builder = AddColumnRequest.newBuilder();
+ for (Mutation m : tableMetaData) {
+ MutationProto mp = ProtobufUtil.toProto(m);
+ builder.addTableMetadataMutations(mp.toByteString());
+ }
+ builder.setClientVersion(VersionUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER));
+ instance.addColumn(controller, builder.build(), rpcCallback);
+ if(controller.getFailedOn() != null) {
+ throw controller.getFailedOn();
+ }
+ return rpcCallback.get();
+ }
+ });
+
+ if (result.getMutationCode() == MutationCode.COLUMN_NOT_FOUND || result.getMutationCode() == MutationCode.TABLE_ALREADY_EXISTS) { // Success
+ success = true;
+ // Flush the table if transitioning DISABLE_WAL from TRUE to FALSE
+ if ( MetaDataUtil.getMutationValue(m,PhoenixDatabaseMetaData.DISABLE_WAL_BYTES, kvBuilder, ptr)
+ && Boolean.FALSE.equals(PBoolean.INSTANCE.toObject(ptr))) {
+ flushTable(table.getPhysicalName().getBytes());
+ }
+
+ if (tableType == PTableType.TABLE) {
+ // If we're changing MULTI_TENANT to true or false, create or drop the view index table
+ if (MetaDataUtil.getMutationValue(m, PhoenixDatabaseMetaData.MULTI_TENANT_BYTES, kvBuilder, ptr)){
+ long timestamp = MetaDataUtil.getClientTimeStamp(m);
+ if (Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(ptr.get(), ptr.getOffset(), ptr.getLength()))) {
+ this.ensureViewIndexTableCreated(table, timestamp);
+ } else {
+ this.ensureViewIndexTableDropped(table.getPhysicalName().getBytes(), timestamp);
+ }
+ }
}
- boolean pollingNotNeeded = (!tableProps.isEmpty() && !existingColumnFamilies(table).containsAll(colFamiliesForPColumnsToBeAdded));
- if (modifyHTable) {
- modifyTable(table.getPhysicalName().getBytes(), tableDescriptor, !pollingNotNeeded);
+ }
+ } finally {
+ // If we weren't successful with our metadata update
+ // and we've already pushed the HBase metadata changes to the server
+ // and we've tried to go from non transactional to transactional
+ // then we must undo the metadata change otherwise the table will
+ // no longer function correctly.
+ // Note that if this fails, we're in a corrupt state.
+ if (!success && metaDataUpdated && nonTxToTx) {
+ sendHBaseMetaData(origTableDescriptors, pollingNeeded);
+ }
+ }
+ return result;
+ }
+ private void updateDescriptorForTx(PTable table, Map<String, Object> tableProps, HTableDescriptor tableDescriptor,
+ String txValue, Set<HTableDescriptor> descriptorsToUpdate, Set<HTableDescriptor> origDescriptors) throws SQLException {
+ HBaseAdmin admin = null;
+ byte[] physicalTableName = table.getPhysicalName().getBytes();
+ try {
+ admin = new HBaseAdmin(config);
+ setTransactional(tableDescriptor, table.getType(), txValue, tableProps);
+ Map<String, Object> indexTableProps;
+ if (txValue == null) {
+ indexTableProps = Collections.<String,Object>emptyMap();
+ } else {
+ indexTableProps = Maps.newHashMapWithExpectedSize(1);
+ indexTableProps.put(TxConstants.READ_NON_TX_DATA, Boolean.valueOf(txValue));
+ }
+ for (PTable index : table.getIndexes()) {
+ HTableDescriptor indexDescriptor = admin.getTableDescriptor(index.getPhysicalName().getBytes());
+ origDescriptors.add(indexDescriptor);
+ indexDescriptor = new HTableDescriptor(indexDescriptor);
+ descriptorsToUpdate.add(indexDescriptor);
+ if (index.getColumnFamilies().isEmpty()) {
+ byte[] dataFamilyName = SchemaUtil.getEmptyColumnFamily(table);
+ byte[] indexFamilyName = SchemaUtil.getEmptyColumnFamily(index);
+ HColumnDescriptor indexColDescriptor = indexDescriptor.getFamily(indexFamilyName);
+ HColumnDescriptor tableColDescriptor = tableDescriptor.getFamily(dataFamilyName);
+ indexColDescriptor.setMaxVersions(tableColDescriptor.getMaxVersions());
+ indexColDescriptor.setValue(TxConstants.PROPERTY_TTL, tableColDescriptor.getValue(TxConstants.PROPERTY_TTL));
+ } else {
+ for (PColumnFamily family : index.getColumnFamilies()) {
+ byte[] familyName = family.getName().getBytes();
+ indexDescriptor.getFamily(familyName).setMaxVersions(tableDescriptor.getFamily(familyName).getMaxVersions());
+ HColumnDescriptor indexColDescriptor = indexDescriptor.getFamily(familyName);
+ HColumnDescriptor tableColDescriptor = tableDescriptor.getFamily(familyName);
+ indexColDescriptor.setMaxVersions(tableColDescriptor.getMaxVersions());
+ indexColDescriptor.setValue(TxConstants.PROPERTY_TTL, tableColDescriptor.getValue(TxConstants.PROPERTY_TTL));
+ }
}
+ setTransactional(indexDescriptor, index.getType(), txValue, indexTableProps);
+ }
+ try {
+ HTableDescriptor indexDescriptor = admin.getTableDescriptor(MetaDataUtil.getViewIndexPhysicalName(physicalTableName));
+ origDescriptors.add(indexDescriptor);
+ indexDescriptor = new HTableDescriptor(indexDescriptor);
+ descriptorsToUpdate.add(indexDescriptor);
+ setSharedIndexMaxVersion(table, tableDescriptor, indexDescriptor);
+ setTransactional(indexDescriptor, PTableType.INDEX, txValue, indexTableProps);
+ } catch (org.apache.hadoop.hbase.TableNotFoundException ignore) {
+ // Ignore, as we may never have created a view index table
+ }
+ try {
+ HTableDescriptor indexDescriptor = admin.getTableDescriptor(MetaDataUtil.getLocalIndexPhysicalName(physicalTableName));
+ origDescriptors.add(indexDescriptor);
+ indexDescriptor = new HTableDescriptor(indexDescriptor);
+ descriptorsToUpdate.add(indexDescriptor);
+ setSharedIndexMaxVersion(table, tableDescriptor, indexDescriptor);
+ setTransactional(indexDescriptor, PTableType.INDEX, txValue, indexTableProps);
+ } catch (org.apache.hadoop.hbase.TableNotFoundException ignore) {
+ // Ignore, as we may never have created a view index table
+ }
+ } catch (IOException e) {
+ throw ServerUtil.parseServerException(e);
+ } finally {
+ try {
+ if (admin != null) admin.close();
+ } catch (IOException e) {
+ logger.warn("Could not close admin",e);
+ }
+ }
+ }
+ private void setSharedIndexMaxVersion(PTable table, HTableDescriptor tableDescriptor,
+ HTableDescriptor indexDescriptor) {
+ if (table.getColumnFamilies().isEmpty()) {
+ byte[] familyName = SchemaUtil.getEmptyColumnFamily(table);
+ HColumnDescriptor indexColDescriptor = indexDescriptor.getFamily(familyName);
+ HColumnDescriptor tableColDescriptor = tableDescriptor.getFamily(familyName);
+ indexColDescriptor.setMaxVersions(tableColDescriptor.getMaxVersions());
+ indexColDescriptor.setValue(TxConstants.PROPERTY_TTL, tableColDescriptor.getValue(TxConstants.PROPERTY_TTL));
+ } else {
+ for (PColumnFamily family : table.getColumnFamilies()) {
+ byte[] familyName = family.getName().getBytes();
+ HColumnDescriptor indexColDescriptor = indexDescriptor.getFamily(familyName);
+ if (indexColDescriptor != null) {
+ HColumnDescriptor tableColDescriptor = tableDescriptor.getFamily(familyName);
+ indexColDescriptor.setMaxVersions(tableColDescriptor.getMaxVersions());
+ indexColDescriptor.setValue(TxConstants.PROPERTY_TTL, tableColDescriptor.getValue(TxConstants.PROPERTY_TTL));
+ }
+ }
+ }
+ }
+
+ private void sendHBaseMetaData(Set<HTableDescriptor> tableDescriptors, boolean pollingNeeded) throws SQLException {
+ SQLException sqlE = null;
+ for (HTableDescriptor descriptor : tableDescriptors) {
+ try {
+ modifyTable(descriptor.getName(), descriptor, pollingNeeded);
} catch (IOException e) {
sqlE = ServerUtil.parseServerException(e);
} catch (InterruptedException e) {
@@ -1535,76 +1843,26 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
}
-
- // Special case for call during drop table to ensure that the empty column family exists.
- // In this, case we only include the table header row, as until we add schemaBytes and tableBytes
- // as args to this function, we have no way of getting them in this case.
- // TODO: change to if (tableMetaData.isEmpty()) once we pass through schemaBytes and tableBytes
- // Also, could be used to update property values on ALTER TABLE t SET prop=xxx
- if ((tableMetaData.isEmpty()) || (tableMetaData.size() == 1 && tableMetaData.get(0).isEmpty())) {
- return new MetaDataMutationResult(MutationCode.NO_OP, System.currentTimeMillis(), table);
- }
- byte[][] rowKeyMetaData = new byte[3][];
- PTableType tableType = table.getType();
-
- Mutation m = tableMetaData.get(0);
- byte[] rowKey = m.getRow();
- SchemaUtil.getVarChars(rowKey, rowKeyMetaData);
- byte[] tenantIdBytes = rowKeyMetaData[PhoenixDatabaseMetaData.TENANT_ID_INDEX];
- byte[] schemaBytes = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
- byte[] tableBytes = rowKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
- byte[] tableKey = SchemaUtil.getTableKey(tenantIdBytes, schemaBytes, tableBytes);
-
- ImmutableBytesWritable ptr = new ImmutableBytesWritable();
- MetaDataMutationResult result = metaDataCoprocessorExec(tableKey,
- new Batch.Call<MetaDataService, MetaDataResponse>() {
- @Override
- public MetaDataResponse call(MetaDataService instance) throws IOException {
- ServerRpcController controller = new ServerRpcController();
- BlockingRpcCallback<MetaDataResponse> rpcCallback =
- new BlockingRpcCallback<MetaDataResponse>();
- AddColumnRequest.Builder builder = AddColumnRequest.newBuilder();
- for (Mutation m : tableMetaData) {
- MutationProto mp = ProtobufUtil.toProto(m);
- builder.addTableMetadataMutations(mp.toByteString());
- }
- builder.setClientVersion(VersionUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER));
- instance.addColumn(controller, builder.build(), rpcCallback);
- if(controller.getFailedOn() != null) {
- throw controller.getFailedOn();
- }
- return rpcCallback.get();
- }
- });
-
- if (result.getMutationCode() == MutationCode.COLUMN_NOT_FOUND) { // Success
- // Flush the table if transitioning DISABLE_WAL from TRUE to FALSE
- if ( MetaDataUtil.getMutationValue(m,PhoenixDatabaseMetaData.DISABLE_WAL_BYTES, kvBuilder, ptr)
- && Boolean.FALSE.equals(PBoolean.INSTANCE.toObject(ptr))) {
- flushTable(table.getPhysicalName().getBytes());
- }
-
- if (tableType == PTableType.TABLE) {
- // If we're changing MULTI_TENANT to true or false, create or drop the view index table
- if (MetaDataUtil.getMutationValue(m, PhoenixDatabaseMetaData.MULTI_TENANT_BYTES, kvBuilder, ptr)){
- long timestamp = MetaDataUtil.getClientTimeStamp(m);
- if (Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(ptr.get(), ptr.getOffset(), ptr.getLength()))) {
- this.ensureViewIndexTableCreated(table, timestamp);
- } else {
- this.ensureViewIndexTableDropped(table.getPhysicalName().getBytes(), timestamp);
- }
- }
- }
+ }
+ private void setTransactional(HTableDescriptor tableDescriptor, PTableType tableType, String txValue, Map<String, Object> tableProps) throws SQLException {
+ if (txValue == null) {
+ tableDescriptor.remove(TxConstants.READ_NON_TX_DATA);
+ } else {
+ tableDescriptor.setValue(TxConstants.READ_NON_TX_DATA, txValue);
}
- return result;
+ this.addCoprocessors(tableDescriptor.getName(), tableDescriptor, tableType, tableProps);
}
- private HTableDescriptor separateAndValidateProperties(PTable table, Map<String, List<Pair<String, Object>>> properties, Set<String> colFamiliesForPColumnsToBeAdded, Map<String, Object> tableProps) throws SQLException {
+ private Pair<HTableDescriptor,HTableDescriptor> separateAndValidateProperties(PTable table, Map<String, List<Pair<String, Object>>> properties, Set<String> colFamiliesForPColumnsToBeAdded, List<Pair<byte[], Map<String, Object>>> families, Map<String, Object> tableProps) throws SQLException {
Map<String, Map<String, Object>> stmtFamiliesPropsMap = new HashMap<>(properties.size());
Map<String,Object> commonFamilyProps = new HashMap<>();
- boolean addingColumns = colFamiliesForPColumnsToBeAdded != null && colFamiliesForPColumnsToBeAdded.size() > 0;
+ boolean addingColumns = colFamiliesForPColumnsToBeAdded != null && !colFamiliesForPColumnsToBeAdded.isEmpty();
HashSet<String> existingColumnFamilies = existingColumnFamilies(table);
Map<String, Map<String, Object>> allFamiliesProps = new HashMap<>(existingColumnFamilies.size());
+ boolean isTransactional = table.isTransactional();
+ boolean willBeTransactional = false;
+ boolean isOrWillBeTransactional = isTransactional;
+ Integer newTTL = null;
for (String family : properties.keySet()) {
List<Pair<String, Object>> propsList = properties.get(family);
if (propsList != null && propsList.size() > 0) {
@@ -1630,22 +1888,26 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (TableProperty.isPhoenixTableProperty(propName)) {
TableProperty.valueOf(propName).validate(true, !family.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY), table.getType());
if (propName.equals(TTL)) {
+ newTTL = ((Number)prop.getSecond()).intValue();
// Even though TTL is really a HColumnProperty we treat it specially.
// We enforce that all column families have the same TTL.
commonFamilyProps.put(propName, prop.getSecond());
+ } else if (propName.equals(PhoenixDatabaseMetaData.TRANSACTIONAL) && Boolean.TRUE.equals(propValue)) {
+ willBeTransactional = isOrWillBeTransactional = true;
+ tableProps.put(TxConstants.READ_NON_TX_DATA, propValue);
}
} else {
if (isHColumnProperty(propName)) {
if (family.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY)) {
- commonFamilyProps.put(propName, prop.getSecond());
+ commonFamilyProps.put(propName, propValue);
} else {
- colFamilyPropsMap.put(propName, prop.getSecond());
+ colFamilyPropsMap.put(propName, propValue);
}
} else {
// invalid property - neither of HTableProp, HColumnProp or PhoenixTableProp
// FIXME: This isn't getting triggered as currently a property gets evaluated
// as HTableProp if its neither HColumnProp or PhoenixTableProp.
- throw new SQLExceptionInfo.Builder(SQLExceptionCode.SET_UNSUPPORTED_PROP_ON_ALTER_TABLE)
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_PROPERTY)
.setMessage("Column Family: " + family + ", Property: " + propName).build()
.buildException();
}
@@ -1728,16 +1990,17 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
allFamiliesProps.put(Bytes.toString(table.getDefaultFamilyName() == null ? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES : table.getDefaultFamilyName().getBytes() ), commonFamilyProps);
}
-
// Views are not allowed to have any of these properties.
if (table.getType() == PTableType.VIEW && (!stmtFamiliesPropsMap.isEmpty() || !commonFamilyProps.isEmpty() || !tableProps.isEmpty())) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.VIEW_WITH_PROPERTIES).build()
.buildException();
}
+
HTableDescriptor newTableDescriptor = null;
+ HTableDescriptor origTableDescriptor = null;
if (!allFamiliesProps.isEmpty() || !tableProps.isEmpty()) {
byte[] tableNameBytes = Bytes.toBytes(table.getPhysicalName().getString());
- HTableDescriptor existingTableDescriptor = getTableDescriptor(tableNameBytes);
+ HTableDescriptor existingTableDescriptor = origTableDescriptor = getTableDescriptor(tableNameBytes);
newTableDescriptor = new HTableDescriptor(existingTableDescriptor);
if (!tableProps.isEmpty()) {
// add all the table properties to the existing table descriptor
@@ -1747,21 +2010,97 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
if (addingColumns) {
// Make sure that all the CFs of the table have the same TTL as the empty CF.
- setTTLToEmptyCFTTL(allFamiliesProps, table, newTableDescriptor);
+ setTTLForNewCFs(allFamiliesProps, table, newTableDescriptor, newTTL);
+ }
+ // Set TTL on all table column families, even if they're not referenced here
+ if (newTTL != null) {
+ for (PColumnFamily family : table.getColumnFamilies()) {
+ if (!allFamiliesProps.containsKey(family.getName().getString())) {
+ Map<String,Object> familyProps = Maps.newHashMapWithExpectedSize(1);
+ familyProps.put(TTL, newTTL);
+ allFamiliesProps.put(family.getName().getString(), familyProps);
+ }
+ }
+ }
+ Integer defaultTxMaxVersions = null;
+ if (isOrWillBeTransactional) {
+ // Calculate default for max versions
+ Map<String, Object> emptyFamilyProps = allFamiliesProps.get(SchemaUtil.getEmptyColumnFamilyAsString(table));
+ if (emptyFamilyProps != null) {
+ defaultTxMaxVersions = (Integer)emptyFamilyProps.get(HConstants.VERSIONS);
+ }
+ if (defaultTxMaxVersions == null) {
+ if (isTransactional) {
+ defaultTxMaxVersions = newTableDescriptor.getFamily(SchemaUtil.getEmptyColumnFamily(table)).getMaxVersions();
+ } else {
+ defaultTxMaxVersions =
+ this.getProps().getInt(
+ QueryServices.MAX_VERSIONS_TRANSACTIONAL_ATTRIB,
+ QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL);
+ }
+ }
+ if (willBeTransactional) {
+ // Set VERSIONS for all column families when transitioning to transactional
+ for (PColumnFamily family : table.getColumnFamilies()) {
+ if (!allFamiliesProps.containsKey(family.getName().getString())) {
+ Map<String,Object> familyProps = Maps.newHashMapWithExpectedSize(1);
+ familyProps.put(HConstants.VERSIONS, defaultTxMaxVersions);
+ allFamiliesProps.put(family.getName().getString(), familyProps);
+ }
+ }
+ }
+ }
+ // Set Tephra's TTL property based on HBase property if we're
+ // transitioning to become transactional or setting TTL on
+ // an already transactional table.
+ if (isOrWillBeTransactional) {
+ int ttl = getTTL(table, newTableDescriptor, newTTL);
+ if (ttl != HColumnDescriptor.DEFAULT_TTL) {
+ for (Map.Entry<String, Map<String, Object>> entry : allFamiliesProps.entrySet()) {
+ Map<String, Object> props = entry.getValue();
+ if (props == null) {
+ props = new HashMap<String, Object>();
+ }
+ props.put(TxConstants.PROPERTY_TTL, ttl);
+ // Remove HBase TTL if we're not transitioning an existing table to become transactional
+ // or if the existing transactional table wasn't originally non transactional.
+ if (!willBeTransactional && !Boolean.valueOf(newTableDescriptor.getValue(TxConstants.READ_NON_TX_DATA))) {
+ props.remove(TTL);
+ }
+ }
+ }
}
for (Entry<String, Map<String, Object>> entry : allFamiliesProps.entrySet()) {
- byte[] cf = entry.getKey().getBytes();
+ Map<String,Object> familyProps = entry.getValue();
+ if (isOrWillBeTransactional) {
+ if (!familyProps.containsKey(HConstants.VERSIONS)) {
+ familyProps.put(HConstants.VERSIONS, defaultTxMaxVersions);
+ }
+ }
+ byte[] cf = Bytes.toBytes(entry.getKey());
HColumnDescriptor colDescriptor = newTableDescriptor.getFamily(cf);
if (colDescriptor == null) {
// new column family
- colDescriptor = generateColumnFamilyDescriptor(new Pair<>(cf, entry.getValue()), table.getType());
+ colDescriptor = generateColumnFamilyDescriptor(new Pair<>(cf, familyProps), table.getType());
newTableDescriptor.addFamily(colDescriptor);
} else {
- modifyColumnFamilyDescriptor(colDescriptor, entry.getValue());
+ modifyColumnFamilyDescriptor(colDescriptor, familyProps);
+ }
+ if (isOrWillBeTransactional) {
+ checkTransactionalVersionsValue(colDescriptor);
}
}
}
- return newTableDescriptor;
+ return new Pair<>(origTableDescriptor, newTableDescriptor);
+ }
+
+ private void checkTransactionalVersionsValue(HColumnDescriptor colDescriptor) throws SQLException {
+ int maxVersions = colDescriptor.getMaxVersions();
+ if (maxVersions <= 1) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE)
+ .setFamilyName(colDescriptor.getNameAsString())
+ .build().buildException();
+ }
}
private boolean isHColumnProperty(String propName) {
@@ -1775,7 +2114,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private HashSet<String> existingColumnFamiliesForBaseTable(PName baseTableName) throws TableNotFoundException {
synchronized (latestMetaDataLock) {
throwConnectionClosedIfNullMetaData();
- PTable table = latestMetaData.getTable(new PTableKey(null, baseTableName.getString()));
+ PTable table = latestMetaData.getTableRef(new PTableKey(null, baseTableName.getString())).getTable();
latestMetaDataLock.notifyAll();
return existingColumnFamilies(table);
}
@@ -1790,20 +2129,23 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
return cfNames;
}
- private int getTTLForEmptyCf(byte[] emptyCf, byte[] tableNameBytes, HTableDescriptor tableDescriptor) throws SQLException {
- if (tableDescriptor == null) {
- tableDescriptor = getTableDescriptor(tableNameBytes);
- }
- return tableDescriptor.getFamily(emptyCf).getTimeToLive();
+ private static int getTTL(PTable table, HTableDescriptor tableDesc, Integer newTTL) throws SQLException {
+ // If we're setting TTL now, then use that value. Otherwise, use empty column family value
+ int ttl = newTTL != null ? newTTL
+ : tableDesc.getFamily(SchemaUtil.getEmptyColumnFamily(table)).getTimeToLive();
+ return ttl;
}
- private void setTTLToEmptyCFTTL(Map<String, Map<String, Object>> familyProps, PTable table,
- HTableDescriptor tableDesc) throws SQLException {
+ private static void setTTLForNewCFs(Map<String, Map<String, Object>> familyProps, PTable table,
+ HTableDescriptor tableDesc, Integer newTTL) throws SQLException {
if (!familyProps.isEmpty()) {
- int emptyCFTTL = getTTLForEmptyCf(SchemaUtil.getEmptyColumnFamily(table), table.getPhysicalName().getBytes(), tableDesc);
- for (String family : familyProps.keySet()) {
- Map<String, Object> props = familyProps.get(family) != null ? familyProps.get(family) : new HashMap<String, Object>();
- props.put(TTL, emptyCFTTL);
+ int ttl = getTTL(table, tableDesc, newTTL);
+ for (Map.Entry<String, Map<String, Object>> entry : familyProps.entrySet()) {
+ Map<String, Object> props = entry.getValue();
+ if (props == null) {
+ props = new HashMap<String, Object>();
+ }
+ props.put(TTL, ttl);
}
}
}
@@ -1860,7 +2202,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
Properties props = PropertiesUtil.deepCopy(oldMetaConnection.getClientInfo());
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp));
// Cannot go through DriverManager or you end up in an infinite loop because it'll call init again
- PhoenixConnection metaConnection = new PhoenixConnection(this, oldMetaConnection.getURL(), props, oldMetaConnection.getMetaDataCache());
+ PhoenixConnection metaConnection = new PhoenixConnection(oldMetaConnection, this, props);
SQLException sqlE = null;
try {
metaConnection.createStatement().executeUpdate("ALTER TABLE " + tableName + " ADD " + (addIfNotExists ? " IF NOT EXISTS " : "") + columns );
@@ -1938,7 +2280,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
String columnsToAdd = "";
if(currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0) {
// We know that we always need to add the STORE_NULLS column for 4.3 release
- columnsToAdd = ", " + PhoenixDatabaseMetaData.STORE_NULLS + " " + PBoolean.INSTANCE.getSqlTypeName();
+ columnsToAdd += "," + PhoenixDatabaseMetaData.STORE_NULLS + " " + PBoolean.INSTANCE.getSqlTypeName();
HBaseAdmin admin = null;
try {
admin = getAdmin();
@@ -2020,6 +2362,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
metaConnection = addColumn(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, columnsToAdd, false);
}
+ if(currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) {
+ columnsToAdd = PhoenixDatabaseMetaData.TRANSACTIONAL + " " + PBoolean.INSTANCE.getSqlTypeName();
+ metaConnection = addColumn(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0, columnsToAdd, false);
+ }
}
int nSaltBuckets = ConnectionQueryServicesImpl.this.props.getInt(QueryServices.SEQUENCE_SALT_BUCKETS_ATTRIB,
@@ -2138,8 +2485,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
@Override
- public boolean hasInvalidIndexConfiguration() {
- return hasInvalidIndexConfiguration;
+ public boolean isMutableIndexWALCodecInstalled() {
+ return isMutableIndexWALCodecInstalled;
}
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 87fb88e..77e4ba9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -24,9 +24,11 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
@@ -81,6 +83,10 @@ import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.SequenceUtil;
+import co.cask.tephra.TransactionManager;
+import co.cask.tephra.TransactionSystemClient;
+import co.cask.tephra.inmemory.InMemoryTxSystemClient;
+
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -99,18 +105,37 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
private PMetaData metaData;
private final Map<SequenceKey, SequenceInfo> sequenceMap = Maps.newHashMap();
private final String userName;
+ private final TransactionSystemClient txSystemClient;
private KeyValueBuilder kvBuilder;
private volatile boolean initialized;
private volatile SQLException initializationException;
private final Map<String, List<HRegionLocation>> tableSplits = Maps.newHashMap();
- public ConnectionlessQueryServicesImpl(QueryServices queryServices, ConnectionInfo connInfo) {
- super(queryServices);
+ public ConnectionlessQueryServicesImpl(QueryServices services, ConnectionInfo connInfo, Properties info) {
+ super(services);
userName = connInfo.getPrincipal();
metaData = newEmptyMetaData();
// Use KeyValueBuilder that builds real KeyValues, as our test utils require this
this.kvBuilder = GenericKeyValueBuilder.INSTANCE;
+ Configuration config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration();
+ for (Entry<String,String> entry : services.getProps()) {
+ config.set(entry.getKey(), entry.getValue());
+ }
+ if (info != null) {
+ for (Object key : info.keySet()) {
+ config.set((String) key, info.getProperty((String) key));
+ }
+ }
+ for (Entry<String,String> entry : connInfo.asProps()) {
+ config.set(entry.getKey(), entry.getValue());
+ }
+
+ // Without making a copy of the configuration we cons up, we lose some of our properties
+ // on the server side during testing.
+ config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration(config);
+ TransactionManager txnManager = new TransactionManager(config);
+ this.txSystemClient = new InMemoryTxSystemClient(txnManager);
}
private PMetaData newEmptyMetaData() {
@@ -141,14 +166,19 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
}
@Override
- public PMetaData addTable(PTable table) throws SQLException {
- return metaData = metaData.addTable(table);
+ public PMetaData addTable(PTable table, long resolvedTime) throws SQLException {
+ return metaData = metaData.addTable(table, resolvedTime);
+ }
+
+ @Override
+ public PMetaData updateResolvedTimestamp(PTable table, long resolvedTimestamp) throws SQLException {
+ return metaData = metaData.updateResolvedTimestamp(table, resolvedTimestamp);
}
@Override
public PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columns, long tableTimeStamp,
- long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls) throws SQLException {
- return metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls);
+ long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException {
+ return metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime);
}
@Override
@@ -159,8 +189,8 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
@Override
public PMetaData removeColumn(PName tenantId, String tableName, List<PColumn> columnsToRemove, long tableTimeStamp,
- long tableSeqNum) throws SQLException {
- return metaData = metaData.removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum);
+ long tableSeqNum, long resolvedTime) throws SQLException {
+ return metaData = metaData.removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum, resolvedTime);
}
@@ -175,7 +205,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
// to get anything from the server (since we don't have a connection)
try {
String fullTableName = SchemaUtil.getTableName(schemaBytes, tableBytes);
- PTable table = metaData.getTable(new PTableKey(tenantId, fullTableName));
+ PTable table = metaData.getTableRef(new PTableKey(tenantId, fullTableName)).getTable();
return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, 0, table, true);
} catch (TableNotFoundException e) {
return new MetaDataMutationResult(MutationCode.TABLE_NOT_FOUND, 0, null);
@@ -345,7 +375,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
String indexName = Bytes.toString(rowKeyMetadata[PhoenixDatabaseMetaData.TABLE_NAME_INDEX]);
String indexTableName = SchemaUtil.getTableName(schemaName, indexName);
PName tenantId = tenantIdBytes.length == 0 ? null : PNameFactory.newName(tenantIdBytes);
- PTable index = metaData.getTable(new PTableKey(tenantId, indexTableName));
+ PTable index = metaData.getTableRef(new PTableKey(tenantId, indexTableName)).getTable();
index = PTableImpl.makePTable(index,newState == PIndexState.USABLE ? PIndexState.ACTIVE : newState == PIndexState.UNUSABLE ? PIndexState.INACTIVE : newState);
return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, 0, index);
}
@@ -360,8 +390,8 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
}
@Override
- public boolean hasInvalidIndexConfiguration() {
- return false;
+ public boolean isMutableIndexWALCodecInstalled() {
+ return true;
}
@Override
@@ -492,6 +522,10 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
}
@Override
+ public TransactionSystemClient getTransactionSystemClient() {
+ return txSystemClient;
+ }
+
public MetaDataMutationResult createFunction(List<Mutation> functionData, PFunction function, boolean temporary)
throws SQLException {
return new MetaDataMutationResult(MutationCode.FUNCTION_NOT_FOUND, 0l, null);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index 4153652..4952355 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -46,6 +46,8 @@ import org.apache.phoenix.schema.SequenceAllocation;
import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.stats.PTableStats;
+import co.cask.tephra.TransactionSystemClient;
+
public class DelegateConnectionQueryServices extends DelegateQueryServices implements ConnectionQueryServices {
@@ -74,14 +76,19 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
}
@Override
- public PMetaData addTable(PTable table) throws SQLException {
- return getDelegate().addTable(table);
+ public PMetaData addTable(PTable table, long resolvedTime) throws SQLException {
+ return getDelegate().addTable(table, resolvedTime);
+ }
+
+ @Override
+ public PMetaData updateResolvedTimestamp(PTable table, long resolvedTimestamp) throws SQLException {
+ return getDelegate().updateResolvedTimestamp(table, resolvedTimestamp);
}
@Override
public PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columns, long tableTimeStamp,
- long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls) throws SQLException {
- return getDelegate().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls);
+ long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException {
+ return getDelegate().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime);
}
@Override
@@ -92,8 +99,8 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
@Override
public PMetaData removeColumn(PName tenantId, String tableName, List<PColumn> columnsToRemove, long tableTimeStamp,
- long tableSeqNum) throws SQLException {
- return getDelegate().removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum);
+ long tableSeqNum, long resolvedTime) throws SQLException {
+ return getDelegate().removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum, resolvedTime);
}
@Override
@@ -165,8 +172,8 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
}
@Override
- public boolean hasInvalidIndexConfiguration() {
- return getDelegate().hasInvalidIndexConfiguration();
+ public boolean isMutableIndexWALCodecInstalled() {
+ return getDelegate().isMutableIndexWALCodecInstalled();
}
@Override
@@ -254,6 +261,10 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
}
@Override
+ public TransactionSystemClient getTransactionSystemClient() {
+ return getDelegate().getTransactionSystemClient();
+ }
+
public MetaDataMutationResult createFunction(List<Mutation> functionData, PFunction function, boolean temporary)
throws SQLException {
return getDelegate().createFunction(functionData, function, temporary);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java b/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java
index 76e7593..8e7a70d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java
@@ -35,10 +35,11 @@ import org.apache.phoenix.schema.PTable;
* @since 0.1
*/
public interface MetaDataMutated {
- PMetaData addTable(PTable table) throws SQLException;
+ PMetaData addTable(PTable table, long resolvedTime) throws SQLException;
+ PMetaData updateResolvedTimestamp(PTable table, long resolvedTimestamp) throws SQLException;
PMetaData removeTable(PName tenantId, String tableName, String parentTableName, long tableTimeStamp) throws SQLException;
- PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls) throws SQLException;
- PMetaData removeColumn(PName tenantId, String tableName, List<PColumn> columnsToRemove, long tableTimeStamp, long tableSeqNum) throws SQLException;
+ PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException;
+ PMetaData removeColumn(PName tenantId, String tableName, List<PColumn> columnsToRemove, long tableTimeStamp, long tableSeqNum, long resolvedTime) throws SQLException;
PMetaData addFunction(PFunction function) throws SQLException;
PMetaData removeFunction(PName tenantId, String function, long functionTimeStamp) throws SQLException;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index 268bfc1..1cac37f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -96,6 +96,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE;
@@ -112,6 +113,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.schema.MetaDataSplitPolicy;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PNameFactory;
@@ -172,9 +174,15 @@ public interface QueryConstants {
public final static int MILLIS_IN_DAY = 1000 * 60 * 60 * 24;
public static final String EMPTY_COLUMN_NAME = "_0";
+ // For transactional tables, the value of our empty key value can no longer be empty
+ // since empty values are treated as column delete markers.
public static final byte[] EMPTY_COLUMN_BYTES = Bytes.toBytes(EMPTY_COLUMN_NAME);
public static final ImmutableBytesPtr EMPTY_COLUMN_BYTES_PTR = new ImmutableBytesPtr(
EMPTY_COLUMN_BYTES);
+ public final static String EMPTY_COLUMN_VALUE = "x";
+ public final static byte[] EMPTY_COLUMN_VALUE_BYTES = Bytes.toBytes(EMPTY_COLUMN_VALUE);
+ public static final ImmutableBytesPtr EMPTY_COLUMN_VALUE_BYTES_PTR = new ImmutableBytesPtr(
+ EMPTY_COLUMN_VALUE_BYTES);
public static final String DEFAULT_COLUMN_FAMILY = "0";
public static final byte[] DEFAULT_COLUMN_FAMILY_BYTES = Bytes.toBytes(DEFAULT_COLUMN_FAMILY);
@@ -251,13 +259,15 @@ public interface QueryConstants {
STORE_NULLS + " BOOLEAN," +
BASE_COLUMN_COUNT + " INTEGER," +
// Column metadata (will be null for table row)
- IS_ROW_TIMESTAMP + " BOOLEAN, " +
+ IS_ROW_TIMESTAMP + " BOOLEAN, " +
+ TRANSACTIONAL + " BOOLEAN," +
"CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ","
+ TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" +
HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
HColumnDescriptor.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n" +
// Install split policy to prevent a tenant's metadata from being split across regions.
- HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "'\n";
+ HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" +
+ PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
public static final String CREATE_STATS_TABLE_METADATA =
"CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_STATS_TABLE + "\"(\n" +
@@ -278,7 +288,8 @@ public interface QueryConstants {
HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_STAT_DATA_VERSIONS + ",\n" +
HColumnDescriptor.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n" +
// Install split policy to prevent a physical table's stats from being split across regions.
- HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "'\n";
+ HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" +
+ PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
public static final String CREATE_SEQUENCE_METADATA =
"CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + TYPE_SEQUENCE + "\"(\n" +
@@ -296,7 +307,8 @@ public interface QueryConstants {
LIMIT_REACHED_FLAG + " BOOLEAN \n" +
" CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + SEQUENCE_SCHEMA + "," + SEQUENCE_NAME + "))\n" +
HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
- HColumnDescriptor.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + "\n";
+ HColumnDescriptor.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n" +
+ PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
public static final String CREATE_FUNCTION_METADATA =
"CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_FUNCTION_TABLE + "\"(\n" +
@@ -320,6 +332,7 @@ public interface QueryConstants {
HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
HColumnDescriptor.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n"+
// Install split policy to prevent a tenant's metadata from being split across regions.
- HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "'\n";
+ HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" +
+ PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index df8cdc3..5f42ff8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -159,6 +159,7 @@ public interface QueryServices extends SQLCloseable {
public static final String DELAY_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.delay";
public static final String DEFAULT_KEEP_DELETED_CELLS_ATTRIB = "phoenix.table.default.keep.deleted.cells";
public static final String DEFAULT_STORE_NULLS_ATTRIB = "phoenix.table.default.store.nulls";
+ public static final String DEFAULT_TRANSACTIONAL_ATTRIB = "phoenix.transactions.default.enabled";
public static final String GLOBAL_METRICS_ENABLED = "phoenix.query.global.metrics.enabled";
// rpc queue configs
@@ -172,6 +173,8 @@ public interface QueryServices extends SQLCloseable {
public static final String RETURN_SEQUENCE_VALUES_ATTRIB = "phoenix.sequence.returnValues";
public static final String EXTRA_JDBC_ARGUMENTS_ATTRIB = "phoenix.jdbc.extra.arguments";
+ public static final String MAX_VERSIONS_TRANSACTIONAL_ATTRIB = "phoenix.transactions.maxVersions";
+
// queryserver configuration keys
public static final String QUERY_SERVER_SERIALIZATION_ATTRIB = "phoenix.queryserver.serialization";
public static final String QUERY_SERVER_META_FACTORY_ATTRIB = "phoenix.queryserver.metafactory.class";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 0434ac9..8c2f895 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -116,7 +116,7 @@ public class QueryServicesOptions {
public static final int DEFAULT_SCAN_CACHE_SIZE = 1000;
public static final int DEFAULT_MAX_INTRA_REGION_PARALLELIZATION = DEFAULT_MAX_QUERY_CONCURRENCY;
public static final int DEFAULT_DISTINCT_VALUE_COMPRESS_THRESHOLD = 1024 * 1024 * 1; // 1 Mb
- public static final int DEFAULT_INDEX_MUTATE_BATCH_SIZE_THRESHOLD = 5;
+ public static final int DEFAULT_INDEX_MUTATE_BATCH_SIZE_THRESHOLD = 3;
public static final long DEFAULT_MAX_SPOOL_TO_DISK_BYTES = 1024000000;
// Only the first chunked batches are fetched in parallel, so this default
// should be on the relatively bigger side of things. Bigger means more
@@ -190,7 +190,9 @@ public class QueryServicesOptions {
public static final boolean DEFAULT_STORE_NULLS = false;
// TODO Change this to true as part of PHOENIX-1543
+ // We'll also need this for transactions to work correctly
public static final boolean DEFAULT_AUTO_COMMIT = false;
+ public static final boolean DEFAULT_TRANSACTIONAL = false;
public static final boolean DEFAULT_IS_GLOBAL_METRICS_ENABLED = true;
private static final String DEFAULT_CLIENT_RPC_CONTROLLER_FACTORY = ClientRpcControllerFactory.class.getName();
@@ -202,6 +204,7 @@ public class QueryServicesOptions {
public static final boolean DEFAULT_ALLOW_USER_DEFINED_FUNCTIONS = false;
public static final boolean DEFAULT_REQUEST_LEVEL_METRICS_ENABLED = false;
public static final boolean DEFAULT_ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE = true;
+ public static final int DEFAULT_MAX_VERSIONS_TRANSACTIONAL = Integer.MAX_VALUE;
public static final boolean DEFAULT_RETURN_SEQUENCE_VALUES = false;
public static final String DEFAULT_EXTRA_JDBC_ARGUMENTS = "";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java
index 5a0d63b..7fb90a1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java
@@ -162,8 +162,8 @@ public class DelegateTable implements PTable {
}
@Override
- public void getIndexMaintainers(ImmutableBytesWritable ptr, PhoenixConnection connection) {
- delegate.getIndexMaintainers(ptr, connection);
+ public boolean getIndexMaintainers(ImmutableBytesWritable ptr, PhoenixConnection connection) {
+ return delegate.getIndexMaintainers(ptr, connection);
}
@Override
@@ -238,6 +238,10 @@ public class DelegateTable implements PTable {
}
@Override
+ public boolean isTransactional() {
+ return delegate.isTransactional();
+ }
+
public int getBaseColumnCount() {
return delegate.getBaseColumnCount();
}
[6/9] phoenix git commit: PHOENIX-1674 Snapshot isolation transaction
support through Tephra (James Taylor, Thomas D'Silva)
Posted by td...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java
index 54c688a..65a43de 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java
@@ -26,7 +26,7 @@ import java.sql.SQLException;
import java.util.List;
import java.util.Set;
-import com.google.common.collect.Sets;
+import co.cask.tephra.Transaction;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.io.WritableUtils;
import org.apache.phoenix.cache.GlobalCache;
import org.apache.phoenix.cache.TenantCache;
+import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.execute.TupleProjector;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.expression.KeyValueColumnExpression;
@@ -58,8 +59,10 @@ import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.ServerUtil;
+import org.apache.phoenix.util.TransactionUtil;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
@@ -191,6 +194,7 @@ public class ScanRegionObserver extends BaseScannerRegionObserver {
Region dataRegion = null;
IndexMaintainer indexMaintainer = null;
byte[][] viewConstants = null;
+ Transaction tx = null;
ColumnReference[] dataColumns = IndexUtil.deserializeDataTableColumnsToJoin(scan);
if (dataColumns != null) {
tupleProjector = IndexUtil.getTupleProjector(scan, dataColumns);
@@ -199,14 +203,16 @@ public class ScanRegionObserver extends BaseScannerRegionObserver {
List<IndexMaintainer> indexMaintainers = localIndexBytes == null ? null : IndexMaintainer.deserialize(localIndexBytes);
indexMaintainer = indexMaintainers.get(0);
viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan);
+ byte[] txState = scan.getAttribute(BaseScannerRegionObserver.TX_STATE);
+ tx = MutationState.decodeTransaction(txState);
}
final TupleProjector p = TupleProjector.deserializeProjectorFromScan(scan);
final HashJoinInfo j = HashJoinInfo.deserializeHashJoinFromScan(scan);
innerScanner =
getWrappedScanner(c, innerScanner, arrayKVRefs, arrayFuncRefs, offset, scan,
- dataColumns, tupleProjector, dataRegion, indexMaintainer, viewConstants,
- kvSchema, kvSchemaBitSet, j == null ? p : null, ptr);
+ dataColumns, tupleProjector, dataRegion, indexMaintainer, tx,
+ viewConstants, kvSchema, kvSchemaBitSet, j == null ? p : null, ptr);
final ImmutableBytesWritable tenantId = ScanUtil.getTenantId(scan);
if (j != null) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
index 2f31b08..9f3bdb4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
@@ -26,18 +26,17 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
import org.apache.phoenix.cache.GlobalCache;
import org.apache.phoenix.cache.TenantCache;
-import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
-
import org.apache.phoenix.coprocessor.ServerCachingProtocol.ServerCacheFactory;
import org.apache.phoenix.coprocessor.generated.ServerCachingProtos.AddServerCacheRequest;
import org.apache.phoenix.coprocessor.generated.ServerCachingProtos.AddServerCacheResponse;
import org.apache.phoenix.coprocessor.generated.ServerCachingProtos.RemoveServerCacheRequest;
import org.apache.phoenix.coprocessor.generated.ServerCachingProtos.RemoveServerCacheResponse;
import org.apache.phoenix.coprocessor.generated.ServerCachingProtos.ServerCachingService;
+import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.protobuf.ProtobufUtil;
+import org.apache.phoenix.util.ByteUtil;
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
@@ -66,6 +65,7 @@ public class ServerCachingEndpointImpl extends ServerCachingService implements C
ImmutableBytesWritable cachePtr =
org.apache.phoenix.protobuf.ProtobufUtil
.toImmutableBytesWritable(request.getCachePtr());
+ byte[] txState = request.hasTxState() ? request.getTxState().toByteArray() : ByteUtil.EMPTY_BYTE_ARRAY;
try {
@SuppressWarnings("unchecked")
@@ -73,7 +73,7 @@ public class ServerCachingEndpointImpl extends ServerCachingService implements C
(Class<ServerCacheFactory>) Class.forName(request.getCacheFactory().getClassName());
ServerCacheFactory cacheFactory = serverCacheFactoryClass.newInstance();
tenantCache.addServerCache(new ImmutableBytesPtr(request.getCacheId().toByteArray()),
- cachePtr, cacheFactory);
+ cachePtr, txState, cacheFactory);
} catch (Throwable e) {
ProtobufUtil.setControllerException(controller, new IOException(e));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
index 4fdfe99..b201c8e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
@@ -36,18 +36,19 @@ import org.apache.phoenix.memory.MemoryManager.MemoryChunk;
*/
public interface ServerCachingProtocol {
public static interface ServerCacheFactory extends Writable {
- public Closeable newCache(ImmutableBytesWritable cachePtr, MemoryChunk chunk) throws SQLException;
+ public Closeable newCache(ImmutableBytesWritable cachePtr, byte[] txState, MemoryChunk chunk) throws SQLException;
}
/**
* Add the cache to the region server cache.
* @param tenantId the tenantId or null if not applicable
* @param cacheId unique identifier of the cache
* @param cachePtr pointer to the byte array of the cache
+ * @param txState TODO
* @param cacheFactory factory that converts from byte array to object representation on the server side
* @return true on success and otherwise throws
* @throws SQLException
*/
- public boolean addServerCache(byte[] tenantId, byte[] cacheId, ImmutableBytesWritable cachePtr, ServerCacheFactory cacheFactory) throws SQLException;
+ public boolean addServerCache(byte[] tenantId, byte[] cacheId, ImmutableBytesWritable cachePtr, byte[] txState, ServerCacheFactory cacheFactory) throws SQLException;
/**
* Remove the cache from the region server cache. Called upon completion of
* the operation when cache is no longer needed.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 5133cb2..ffdc05f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -105,6 +105,8 @@ import org.apache.phoenix.util.TimeKeeper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import co.cask.tephra.TxConstants;
+
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -403,6 +405,8 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
Delete delete = new Delete(firstKV.getRowArray(),
firstKV.getRowOffset(), firstKV.getRowLength(),ts);
mutations.add(delete);
+ // force tephra to ignore this deletes
+ delete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
} else if (isUpsert) {
Arrays.fill(values, null);
int i = 0;
@@ -462,6 +466,8 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
results.get(0).getRowOffset(),
results.get(0).getRowLength());
delete.deleteColumns(deleteCF, deleteCQ, ts);
+ // force tephra to ignore this deletes
+ delete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
mutations.add(delete);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
index 25f8271..7e71cd9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
@@ -3218,6 +3218,16 @@ public final class PTableProtos {
* <code>optional bool rowKeyOrderOptimizable = 26;</code>
*/
boolean getRowKeyOrderOptimizable();
+
+ // optional bool transactional = 27;
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ boolean hasTransactional();
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ boolean getTransactional();
}
/**
* Protobuf type {@code PTable}
@@ -3418,6 +3428,11 @@ public final class PTableProtos {
rowKeyOrderOptimizable_ = input.readBool();
break;
}
+ case 216: {
+ bitField0_ |= 0x00400000;
+ transactional_ = input.readBool();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -3980,6 +3995,22 @@ public final class PTableProtos {
return rowKeyOrderOptimizable_;
}
+ // optional bool transactional = 27;
+ public static final int TRANSACTIONAL_FIELD_NUMBER = 27;
+ private boolean transactional_;
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ public boolean hasTransactional() {
+ return ((bitField0_ & 0x00400000) == 0x00400000);
+ }
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ public boolean getTransactional() {
+ return transactional_;
+ }
+
private void initFields() {
schemaNameBytes_ = com.google.protobuf.ByteString.EMPTY;
tableNameBytes_ = com.google.protobuf.ByteString.EMPTY;
@@ -4007,6 +4038,7 @@ public final class PTableProtos {
storeNulls_ = false;
baseColumnCount_ = 0;
rowKeyOrderOptimizable_ = false;
+ transactional_ = false;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -4152,6 +4184,9 @@ public final class PTableProtos {
if (((bitField0_ & 0x00200000) == 0x00200000)) {
output.writeBool(26, rowKeyOrderOptimizable_);
}
+ if (((bitField0_ & 0x00400000) == 0x00400000)) {
+ output.writeBool(27, transactional_);
+ }
getUnknownFields().writeTo(output);
}
@@ -4270,6 +4305,10 @@ public final class PTableProtos {
size += com.google.protobuf.CodedOutputStream
.computeBoolSize(26, rowKeyOrderOptimizable_);
}
+ if (((bitField0_ & 0x00400000) == 0x00400000)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBoolSize(27, transactional_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -4411,6 +4450,11 @@ public final class PTableProtos {
result = result && (getRowKeyOrderOptimizable()
== other.getRowKeyOrderOptimizable());
}
+ result = result && (hasTransactional() == other.hasTransactional());
+ if (hasTransactional()) {
+ result = result && (getTransactional()
+ == other.getTransactional());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -4528,6 +4572,10 @@ public final class PTableProtos {
hash = (37 * hash) + ROWKEYORDEROPTIMIZABLE_FIELD_NUMBER;
hash = (53 * hash) + hashBoolean(getRowKeyOrderOptimizable());
}
+ if (hasTransactional()) {
+ hash = (37 * hash) + TRANSACTIONAL_FIELD_NUMBER;
+ hash = (53 * hash) + hashBoolean(getTransactional());
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -4704,6 +4752,8 @@ public final class PTableProtos {
bitField0_ = (bitField0_ & ~0x01000000);
rowKeyOrderOptimizable_ = false;
bitField0_ = (bitField0_ & ~0x02000000);
+ transactional_ = false;
+ bitField0_ = (bitField0_ & ~0x04000000);
return this;
}
@@ -4852,6 +4902,10 @@ public final class PTableProtos {
to_bitField0_ |= 0x00200000;
}
result.rowKeyOrderOptimizable_ = rowKeyOrderOptimizable_;
+ if (((from_bitField0_ & 0x04000000) == 0x04000000)) {
+ to_bitField0_ |= 0x00400000;
+ }
+ result.transactional_ = transactional_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -5024,6 +5078,9 @@ public final class PTableProtos {
if (other.hasRowKeyOrderOptimizable()) {
setRowKeyOrderOptimizable(other.getRowKeyOrderOptimizable());
}
+ if (other.hasTransactional()) {
+ setTransactional(other.getTransactional());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -6694,6 +6751,39 @@ public final class PTableProtos {
return this;
}
+ // optional bool transactional = 27;
+ private boolean transactional_ ;
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ public boolean hasTransactional() {
+ return ((bitField0_ & 0x04000000) == 0x04000000);
+ }
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ public boolean getTransactional() {
+ return transactional_;
+ }
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ public Builder setTransactional(boolean value) {
+ bitField0_ |= 0x04000000;
+ transactional_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ public Builder clearTransactional() {
+ bitField0_ = (bitField0_ & ~0x04000000);
+ transactional_ = false;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:PTable)
}
@@ -6740,7 +6830,7 @@ public final class PTableProtos {
"leStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030\002 \003(\014\022\033\n\023" +
"guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyBytesCou",
"nt\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n\013pGu" +
- "idePosts\030\006 \001(\0132\014.PGuidePosts\"\357\004\n\006PTable\022" +
+ "idePosts\030\006 \001(\0132\014.PGuidePosts\"\206\005\n\006PTable\022" +
"\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tableNameByt" +
"es\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.PTableType" +
"\022\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenceNumber\030\005" +
@@ -6756,10 +6846,11 @@ public final class PTableProtos {
"wIndexId\030\025 \001(\005\022\021\n\tindexType\030\026 \001(\014\022\026\n\016sta" +
"tsTimeStamp\030\027 \001(\003\022\022\n\nstoreNulls\030\030 \001(\010\022\027\n" +
"\017baseColumnCount\030\031 \001(\005\022\036\n\026rowKeyOrderOpt" +
- "imizable\030\032 \001(\010*A\n\nPTableType\022\n\n\006SYSTEM\020\000" +
- "\022\010\n\004USER\020\001\022\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020" +
- "\004B@\n(org.apache.phoenix.coprocessor.gene",
- "ratedB\014PTableProtosH\001\210\001\001\240\001\001"
+ "imizable\030\032 \001(\010\022\025\n\rtransactional\030\033 \001(\010*A\n" +
+ "\nPTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIE" +
+ "W\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.p",
+ "hoenix.coprocessor.generatedB\014PTableProt" +
+ "osH\001\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -6783,7 +6874,7 @@ public final class PTableProtos {
internal_static_PTable_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_PTable_descriptor,
- new java.lang.String[] { "SchemaNameBytes", "TableNameBytes", "TableType", "IndexState", "SequenceNumber", "TimeStamp", "PkNameBytes", "BucketNum", "Columns", "Indexes", "IsImmutableRows", "GuidePosts", "DataTableNameBytes", "DefaultFamilyName", "DisableWAL", "MultiTenant", "ViewType", "ViewStatement", "PhysicalNames", "TenantId", "ViewIndexId", "IndexType", "StatsTimeStamp", "StoreNulls", "BaseColumnCount", "RowKeyOrderOptimizable", });
+ new java.lang.String[] { "SchemaNameBytes", "TableNameBytes", "TableType", "IndexState", "SequenceNumber", "TimeStamp", "PkNameBytes", "BucketNum", "Columns", "Indexes", "IsImmutableRows", "GuidePosts", "DataTableNameBytes", "DefaultFamilyName", "DisableWAL", "MultiTenant", "ViewType", "ViewStatement", "PhysicalNames", "TenantId", "ViewIndexId", "IndexType", "StatsTimeStamp", "StoreNulls", "BaseColumnCount", "RowKeyOrderOptimizable", "Transactional", });
return null;
}
};
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
index 69db21b..5ee1dfb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
@@ -699,6 +699,16 @@ public final class ServerCachingProtos {
* <code>required .ServerCacheFactory cacheFactory = 4;</code>
*/
org.apache.phoenix.coprocessor.generated.ServerCacheFactoryProtos.ServerCacheFactoryOrBuilder getCacheFactoryOrBuilder();
+
+ // optional bytes txState = 5;
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ boolean hasTxState();
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ com.google.protobuf.ByteString getTxState();
}
/**
* Protobuf type {@code AddServerCacheRequest}
@@ -787,6 +797,11 @@ public final class ServerCachingProtos {
bitField0_ |= 0x00000008;
break;
}
+ case 42: {
+ bitField0_ |= 0x00000010;
+ txState_ = input.readBytes();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -903,11 +918,28 @@ public final class ServerCachingProtos {
return cacheFactory_;
}
+ // optional bytes txState = 5;
+ public static final int TXSTATE_FIELD_NUMBER = 5;
+ private com.google.protobuf.ByteString txState_;
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ public boolean hasTxState() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ public com.google.protobuf.ByteString getTxState() {
+ return txState_;
+ }
+
private void initFields() {
tenantId_ = com.google.protobuf.ByteString.EMPTY;
cacheId_ = com.google.protobuf.ByteString.EMPTY;
cachePtr_ = org.apache.phoenix.coprocessor.generated.ServerCachingProtos.ImmutableBytesWritable.getDefaultInstance();
cacheFactory_ = org.apache.phoenix.coprocessor.generated.ServerCacheFactoryProtos.ServerCacheFactory.getDefaultInstance();
+ txState_ = com.google.protobuf.ByteString.EMPTY;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -953,6 +985,9 @@ public final class ServerCachingProtos {
if (((bitField0_ & 0x00000008) == 0x00000008)) {
output.writeMessage(4, cacheFactory_);
}
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ output.writeBytes(5, txState_);
+ }
getUnknownFields().writeTo(output);
}
@@ -978,6 +1013,10 @@ public final class ServerCachingProtos {
size += com.google.protobuf.CodedOutputStream
.computeMessageSize(4, cacheFactory_);
}
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(5, txState_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -1021,6 +1060,11 @@ public final class ServerCachingProtos {
result = result && getCacheFactory()
.equals(other.getCacheFactory());
}
+ result = result && (hasTxState() == other.hasTxState());
+ if (hasTxState()) {
+ result = result && getTxState()
+ .equals(other.getTxState());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -1050,6 +1094,10 @@ public final class ServerCachingProtos {
hash = (37 * hash) + CACHEFACTORY_FIELD_NUMBER;
hash = (53 * hash) + getCacheFactory().hashCode();
}
+ if (hasTxState()) {
+ hash = (37 * hash) + TXSTATE_FIELD_NUMBER;
+ hash = (53 * hash) + getTxState().hashCode();
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -1177,6 +1225,8 @@ public final class ServerCachingProtos {
cacheFactoryBuilder_.clear();
}
bitField0_ = (bitField0_ & ~0x00000008);
+ txState_ = com.google.protobuf.ByteString.EMPTY;
+ bitField0_ = (bitField0_ & ~0x00000010);
return this;
}
@@ -1229,6 +1279,10 @@ public final class ServerCachingProtos {
} else {
result.cacheFactory_ = cacheFactoryBuilder_.build();
}
+ if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+ to_bitField0_ |= 0x00000010;
+ }
+ result.txState_ = txState_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -1257,6 +1311,9 @@ public final class ServerCachingProtos {
if (other.hasCacheFactory()) {
mergeCacheFactory(other.getCacheFactory());
}
+ if (other.hasTxState()) {
+ setTxState(other.getTxState());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -1610,6 +1667,42 @@ public final class ServerCachingProtos {
return cacheFactoryBuilder_;
}
+ // optional bytes txState = 5;
+ private com.google.protobuf.ByteString txState_ = com.google.protobuf.ByteString.EMPTY;
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ public boolean hasTxState() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ public com.google.protobuf.ByteString getTxState() {
+ return txState_;
+ }
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ public Builder setTxState(com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000010;
+ txState_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ public Builder clearTxState() {
+ bitField0_ = (bitField0_ & ~0x00000010);
+ txState_ = getDefaultInstance().getTxState();
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:AddServerCacheRequest)
}
@@ -3383,20 +3476,21 @@ public final class ServerCachingProtos {
"\n\032ServerCachingService.proto\032\030ServerCach" +
"eFactory.proto\"K\n\026ImmutableBytesWritable" +
"\022\021\n\tbyteArray\030\001 \002(\014\022\016\n\006offset\030\002 \002(\005\022\016\n\006l" +
- "ength\030\003 \002(\005\"\220\001\n\025AddServerCacheRequest\022\020\n" +
+ "ength\030\003 \002(\005\"\241\001\n\025AddServerCacheRequest\022\020\n" +
"\010tenantId\030\001 \001(\014\022\017\n\007cacheId\030\002 \002(\014\022)\n\010cach" +
"ePtr\030\003 \002(\0132\027.ImmutableBytesWritable\022)\n\014c" +
- "acheFactory\030\004 \002(\0132\023.ServerCacheFactory\"(" +
- "\n\026AddServerCacheResponse\022\016\n\006return\030\001 \002(\010" +
- "\"=\n\030RemoveServerCacheRequest\022\020\n\010tenantId" +
- "\030\001 \001(\014\022\017\n\007cacheId\030\002 \002(\014\"+\n\031RemoveServerC",
- "acheResponse\022\016\n\006return\030\001 \002(\0102\245\001\n\024ServerC" +
- "achingService\022A\n\016addServerCache\022\026.AddSer" +
- "verCacheRequest\032\027.AddServerCacheResponse" +
- "\022J\n\021removeServerCache\022\031.RemoveServerCach" +
- "eRequest\032\032.RemoveServerCacheResponseBG\n(" +
- "org.apache.phoenix.coprocessor.generated" +
- "B\023ServerCachingProtosH\001\210\001\001\240\001\001"
+ "acheFactory\030\004 \002(\0132\023.ServerCacheFactory\022\017" +
+ "\n\007txState\030\005 \001(\014\"(\n\026AddServerCacheRespons" +
+ "e\022\016\n\006return\030\001 \002(\010\"=\n\030RemoveServerCacheRe" +
+ "quest\022\020\n\010tenantId\030\001 \001(\014\022\017\n\007cacheId\030\002 \002(\014",
+ "\"+\n\031RemoveServerCacheResponse\022\016\n\006return\030" +
+ "\001 \002(\0102\245\001\n\024ServerCachingService\022A\n\016addSer" +
+ "verCache\022\026.AddServerCacheRequest\032\027.AddSe" +
+ "rverCacheResponse\022J\n\021removeServerCache\022\031" +
+ ".RemoveServerCacheRequest\032\032.RemoveServer" +
+ "CacheResponseBG\n(org.apache.phoenix.copr" +
+ "ocessor.generatedB\023ServerCachingProtosH\001" +
+ "\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -3414,7 +3508,7 @@ public final class ServerCachingProtos {
internal_static_AddServerCacheRequest_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_AddServerCacheRequest_descriptor,
- new java.lang.String[] { "TenantId", "CacheId", "CachePtr", "CacheFactory", });
+ new java.lang.String[] { "TenantId", "CacheId", "CachePtr", "CacheFactory", "TxState", });
internal_static_AddServerCacheResponse_descriptor =
getDescriptor().getMessageTypes().get(2);
internal_static_AddServerCacheResponse_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index bb76ccb..3613c95 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -159,6 +159,12 @@ public enum SQLExceptionCode {
AGGREGATE_EXPRESSION_NOT_ALLOWED_IN_INDEX(520, "42897", "Aggregate expression not allowed in an index"),
NON_DETERMINISTIC_EXPRESSION_NOT_ALLOWED_IN_INDEX(521, "42898", "Non-deterministic expression not allowed in an index"),
STATELESS_EXPRESSION_NOT_ALLOWED_IN_INDEX(522, "42899", "Stateless expression not allowed in an index"),
+
+ /**
+ * Transaction exceptions.
+ */
+ TRANSACTION_CONFLICT_EXCEPTION(523, "42900", "Transaction aborted due to conflict with other mutations"),
+ TRANSACTION_EXCEPTION(524, "42901", "Transaction aborted due to error"),
/**
* Union All related errors
@@ -258,7 +264,18 @@ public enum SQLExceptionCode {
NO_LOCAL_INDEXES(1054, "43A11", "Local secondary indexes are not supported for HBase versions " +
MetaDataUtil.decodeHBaseVersionAsString(PhoenixDatabaseMetaData.MIN_LOCAL_SI_VERSION_DISALLOW) + " through " + MetaDataUtil.decodeHBaseVersionAsString(PhoenixDatabaseMetaData.MAX_LOCAL_SI_VERSION_DISALLOW) + " inclusive."),
UNALLOWED_LOCAL_INDEXES(1055, "43A12", "Local secondary indexes are configured to not be allowed."),
+
DESC_VARBINARY_NOT_SUPPORTED(1056, "43A13", "Descending VARBINARY columns not supported"),
+
+ DEFAULT_COLUMN_FAMILY_ON_SHARED_TABLE(1069, "43A13", "Default column family not allowed on VIEW or shared INDEX"),
+ ONLY_TABLE_MAY_BE_DECLARED_TRANSACTIONAL(1070, "44A01", "Only tables may be declared as transactional"),
+ TX_MAY_NOT_SWITCH_TO_NON_TX(1071, "44A02", "A transactional table may not be switched to non transactional"),
+ STORE_NULLS_MUST_BE_TRUE_FOR_TRANSACTIONAL(1072, "44A03", "Store nulls must be true when a table is transactional"),
+ CANNOT_START_TRANSACTION_WITH_SCN_SET(1073, "44A04", "Cannot start a transaction on a connection with SCN set"),
+ TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE(1074, "44A05", "A transactional table must define VERSION of greater than one"),
+ CANNOT_SPECIFY_SCN_FOR_TXN_TABLE(1075, "44A06", "Cannot use a connection with SCN set for a transactional table"),
+ NULL_TRANSACTION_CONTEXT(1076, "44A07", "No Tranasction Context available"),
+ TRANSACTION_FAILED(1077, "44A08", "Transaction Failure "),
/** Sequence related */
SEQUENCE_ALREADY_EXIST(1200, "42Z00", "Sequence already exists.", new Factory() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
index e292dd8..0bdb65a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
@@ -223,23 +223,26 @@ public abstract class BaseQueryPlan implements QueryPlan {
if (table.getType() != PTableType.SYSTEM) {
scan.setConsistency(connection.getConsistency());
}
- // Get the time range of row_timestamp column
- TimeRange rowTimestampRange = context.getScanRanges().getRowTimestampRange();
- // Get the already existing time range on the scan.
- TimeRange scanTimeRange = scan.getTimeRange();
- Long scn = connection.getSCN();
- if (scn == null) {
- scn = context.getCurrentTime();
- }
- try {
- TimeRange timeRangeToUse = ScanUtil.intersectTimeRange(rowTimestampRange, scanTimeRange, scn);
- if (timeRangeToUse == null) {
- return ResultIterator.EMPTY_ITERATOR;
- }
- scan.setTimeRange(timeRangeToUse.getMin(), timeRangeToUse.getMax());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ // TODO fix this in PHOENIX-2415 Support ROW_TIMESTAMP with transactional tables
+ if (!table.isTransactional()) {
+ // Get the time range of row_timestamp column
+ TimeRange rowTimestampRange = context.getScanRanges().getRowTimestampRange();
+ // Get the already existing time range on the scan.
+ TimeRange scanTimeRange = scan.getTimeRange();
+ Long scn = connection.getSCN();
+ if (scn == null) {
+ scn = context.getCurrentTime();
+ }
+ try {
+ TimeRange timeRangeToUse = ScanUtil.intersectTimeRange(rowTimestampRange, scanTimeRange, scn);
+ if (timeRangeToUse == null) {
+ return ResultIterator.EMPTY_ITERATOR;
+ }
+ scan.setTimeRange(timeRangeToUse.getMin(), timeRangeToUse.getMax());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
byte[] tenantIdBytes;
if( table.isMultiTenant() == true ) {
tenantIdBytes = connection.getTenantId() == null ? null :
@@ -326,6 +329,9 @@ public abstract class BaseQueryPlan implements QueryPlan {
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
IndexMaintainer.serialize(dataTable, ptr, indexes, context.getConnection());
scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD, ByteUtil.copyKeyBytesIfNecessary(ptr));
+ if (dataTable.isTransactional()) {
+ scan.setAttribute(BaseScannerRegionObserver.TX_STATE, context.getConnection().getMutationState().encodeTransaction());
+ }
}
private void serializeViewConstantsIntoScan(Scan scan, PTable dataTable) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java
new file mode 100644
index 0000000..6b3f9ca
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.execute;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.coprocessor.Batch.Call;
+import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
+
+import com.google.protobuf.Descriptors.MethodDescriptor;
+import com.google.protobuf.Message;
+import com.google.protobuf.Service;
+import com.google.protobuf.ServiceException;
+
+public class DelegateHTable implements HTableInterface {
+ protected final HTableInterface delegate;
+
+ public DelegateHTable(HTableInterface delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public byte[] getTableName() {
+ return delegate.getTableName();
+ }
+
+ @Override
+ public TableName getName() {
+ return delegate.getName();
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return delegate.getConfiguration();
+ }
+
+ @Override
+ public HTableDescriptor getTableDescriptor() throws IOException {
+ return delegate.getTableDescriptor();
+ }
+
+ @Override
+ public boolean exists(Get get) throws IOException {
+ return delegate.exists(get);
+ }
+
+ @Override
+ public Boolean[] exists(List<Get> gets) throws IOException {
+ return delegate.exists(gets);
+ }
+
+ @Override
+ public void batch(List<? extends Row> actions, Object[] results) throws IOException, InterruptedException {
+ delegate.batch(actions, results);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public Object[] batch(List<? extends Row> actions) throws IOException, InterruptedException {
+ return delegate.batch(actions);
+ }
+
+ @Override
+ public <R> void batchCallback(List<? extends Row> actions, Object[] results, Callback<R> callback)
+ throws IOException, InterruptedException {
+ delegate.batchCallback(actions, results, callback);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public <R> Object[] batchCallback(List<? extends Row> actions, Callback<R> callback) throws IOException,
+ InterruptedException {
+ return delegate.batchCallback(actions, callback);
+ }
+
+ @Override
+ public Result get(Get get) throws IOException {
+ return delegate.get(get);
+ }
+
+ @Override
+ public Result[] get(List<Get> gets) throws IOException {
+ return delegate.get(gets);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
+ return delegate.getRowOrBefore(row, family);
+ }
+
+ @Override
+ public ResultScanner getScanner(Scan scan) throws IOException {
+ return delegate.getScanner(scan);
+ }
+
+ @Override
+ public ResultScanner getScanner(byte[] family) throws IOException {
+ return delegate.getScanner(family);
+ }
+
+ @Override
+ public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
+ return delegate.getScanner(family, qualifier);
+ }
+
+ @Override
+ public void put(Put put) throws IOException {
+ delegate.put(put);
+ }
+
+ @Override
+ public void put(List<Put> puts) throws IOException {
+ delegate.put(puts);
+ }
+
+ @Override
+ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, byte[] value, Put put) throws IOException {
+ return delegate.checkAndPut(row, family, qualifier, value, put);
+ }
+
+ @Override
+ public void delete(Delete delete) throws IOException {
+ delegate.delete(delete);
+ }
+
+ @Override
+ public void delete(List<Delete> deletes) throws IOException {
+ delegate.delete(deletes);
+ }
+
+ @Override
+ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, byte[] value, Delete delete)
+ throws IOException {
+ return delegate.checkAndDelete(row, family, qualifier, value, delete);
+ }
+
+ @Override
+ public void mutateRow(RowMutations rm) throws IOException {
+ delegate.mutateRow(rm);
+ }
+
+ @Override
+ public Result append(Append append) throws IOException {
+ return delegate.append(append);
+ }
+
+ @Override
+ public Result increment(Increment increment) throws IOException {
+ return delegate.increment(increment);
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
+ return delegate.incrementColumnValue(row, family, qualifier, amount);
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, Durability durability)
+ throws IOException {
+ return delegate.incrementColumnValue(row, family, qualifier, amount, durability);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL)
+ throws IOException {
+ return delegate.incrementColumnValue(row, family, qualifier, amount, writeToWAL);
+ }
+
+ @Override
+ public boolean isAutoFlush() {
+ return delegate.isAutoFlush();
+ }
+
+ @Override
+ public void flushCommits() throws IOException {
+ delegate.flushCommits();
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ }
+
+ @Override
+ public CoprocessorRpcChannel coprocessorService(byte[] row) {
+ return delegate.coprocessorService(row);
+ }
+
+ @Override
+ public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
+ Call<T, R> callable) throws ServiceException, Throwable {
+ return delegate.coprocessorService(service, startKey, endKey, callable);
+ }
+
+ @Override
+ public <T extends Service, R> void coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
+ Call<T, R> callable, Callback<R> callback) throws ServiceException, Throwable {
+ delegate.coprocessorService(service, startKey, endKey, callable, callback);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void setAutoFlush(boolean autoFlush) {
+ delegate.setAutoFlush(autoFlush);
+ }
+
+ @Override
+ public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
+ delegate.setAutoFlush(autoFlush, clearBufferOnFail);
+ }
+
+ @Override
+ public void setAutoFlushTo(boolean autoFlush) {
+ delegate.setAutoFlushTo(autoFlush);
+ }
+
+ @Override
+ public long getWriteBufferSize() {
+ return delegate.getWriteBufferSize();
+ }
+
+ @Override
+ public void setWriteBufferSize(long writeBufferSize) throws IOException {
+ delegate.setWriteBufferSize(writeBufferSize);
+ }
+
+ @Override
+ public <R extends Message> Map<byte[], R> batchCoprocessorService(MethodDescriptor methodDescriptor,
+ Message request, byte[] startKey, byte[] endKey, R responsePrototype) throws ServiceException, Throwable {
+ return delegate.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype);
+ }
+
+ @Override
+ public <R extends Message> void batchCoprocessorService(MethodDescriptor methodDescriptor, Message request,
+ byte[] startKey, byte[] endKey, R responsePrototype, Callback<R> callback) throws ServiceException,
+ Throwable {
+ delegate.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype, callback);
+ }
+
+ @Override
+ public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, CompareOp compareOp, byte[] value,
+ RowMutations mutation) throws IOException {
+ return delegate.checkAndMutate(row, family, qualifier, compareOp, value, mutation);
+ }
+
+ @Override
+ public boolean[] existsAll(List<Get> gets) throws IOException {
+ return delegate.existsAll(gets);
+ }
+
+ @Override
+ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier,
+ CompareOp compareOp, byte[] value, Put put) throws IOException {
+ return delegate.checkAndPut(row, family, qualifier, value, put);
+ }
+
+ @Override
+ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
+ CompareOp compareOp, byte[] value, Delete delete)
+ throws IOException {
+ return delegate.checkAndDelete(row, family, qualifier, compareOp, value, delete);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
index 72920b2..cf89380 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
@@ -23,6 +23,7 @@ import static org.apache.phoenix.util.LogUtil.addCustomAnnotations;
import java.sql.SQLException;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -66,6 +67,7 @@ import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PArrayDataType;
import org.apache.phoenix.schema.types.PBoolean;
@@ -75,6 +77,7 @@ import org.apache.phoenix.util.SQLCloseable;
import org.apache.phoenix.util.SQLCloseables;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
public class HashJoinPlan extends DelegateQueryPlan {
private static final Log LOG = LogFactory.getLog(HashJoinPlan.class);
@@ -83,6 +86,7 @@ public class HashJoinPlan extends DelegateQueryPlan {
private final HashJoinInfo joinInfo;
private final SubPlan[] subPlans;
private final boolean recompileWhereClause;
+ private final Set<TableRef> tableRefs;
private final int maxServerCacheTimeToLive;
private List<SQLCloseable> dependencies;
private HashCacheClient hashClient;
@@ -114,11 +118,21 @@ public class HashJoinPlan extends DelegateQueryPlan {
this.joinInfo = joinInfo;
this.subPlans = subPlans;
this.recompileWhereClause = recompileWhereClause;
+ this.tableRefs = Sets.newHashSetWithExpectedSize(subPlans.length + plan.getSourceRefs().size());
+ this.tableRefs.addAll(plan.getSourceRefs());
+ for (SubPlan subPlan : subPlans) {
+ tableRefs.addAll(subPlan.getInnerPlan().getSourceRefs());
+ }
this.maxServerCacheTimeToLive = plan.getContext().getConnection().getQueryServices().getProps().getInt(
QueryServices.MAX_SERVER_CACHE_TIME_TO_LIVE_MS_ATTRIB, QueryServicesOptions.DEFAULT_MAX_SERVER_CACHE_TIME_TO_LIVE_MS);
}
@Override
+ public Set<TableRef> getSourceRefs() {
+ return tableRefs;
+ }
+
+ @Override
public ResultIterator iterator() throws SQLException {
return iterator(DefaultParallelScanGrouper.getInstance());
}
@@ -251,6 +265,7 @@ public class HashJoinPlan extends DelegateQueryPlan {
public void postProcess(Object result, HashJoinPlan parent) throws SQLException;
public List<String> getPreSteps(HashJoinPlan parent) throws SQLException;
public List<String> getPostSteps(HashJoinPlan parent) throws SQLException;
+ public QueryPlan getInnerPlan();
}
public static class WhereClauseSubPlan implements SubPlan {
@@ -321,6 +336,11 @@ public class HashJoinPlan extends DelegateQueryPlan {
public List<String> getPostSteps(HashJoinPlan parent) throws SQLException {
return Collections.<String>emptyList();
}
+
+ @Override
+ public QueryPlan getInnerPlan() {
+ return plan;
+ }
}
public static class HashSubPlan implements SubPlan {
@@ -411,7 +431,12 @@ public class HashJoinPlan extends DelegateQueryPlan {
+ " IN (" + keyRangeRhsExpression.toString() + ")";
return Collections.<String> singletonList(step);
}
-
+
+
+ @Override
+ public QueryPlan getInnerPlan() {
+ return plan;
+ }
}
}
[5/9] phoenix git commit: PHOENIX-1674 Snapshot isolation transaction
support through Tephra (James Taylor, Thomas D'Silva)
Posted by td...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
index d4ef1cf..3444054 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -29,26 +29,33 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import javax.annotation.Nonnull;
import javax.annotation.concurrent.Immutable;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.htrace.Span;
import org.apache.htrace.TraceScope;
import org.apache.phoenix.cache.ServerCacheClient;
import org.apache.phoenix.cache.ServerCacheClient.ServerCache;
+import org.apache.phoenix.compile.MutationPlan;
+import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.index.IndexMaintainer;
import org.apache.phoenix.index.IndexMetaDataCacheClient;
import org.apache.phoenix.index.PhoenixIndexCodec;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixStatement.Operation;
import org.apache.phoenix.monitoring.GlobalClientMetrics;
import org.apache.phoenix.monitoring.MutationMetricQueue;
import org.apache.phoenix.monitoring.MutationMetricQueue.MutationMetric;
@@ -58,9 +65,9 @@ import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.IllegalDataException;
import org.apache.phoenix.schema.MetaDataClient;
import org.apache.phoenix.schema.PColumn;
-import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PRow;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.RowKeySchema;
import org.apache.phoenix.schema.TableRef;
@@ -72,14 +79,25 @@ import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.LogUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.SQLCloseable;
-import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.ServerUtil;
+import org.apache.phoenix.util.TransactionUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import co.cask.tephra.Transaction;
+import co.cask.tephra.Transaction.VisibilityLevel;
+import co.cask.tephra.TransactionAware;
+import co.cask.tephra.TransactionCodec;
+import co.cask.tephra.TransactionContext;
+import co.cask.tephra.TransactionFailureException;
+import co.cask.tephra.TransactionSystemClient;
+import co.cask.tephra.hbase11.TransactionAwareHTable;
+
+import com.google.common.base.Predicate;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
/**
*
@@ -90,44 +108,217 @@ import com.google.common.collect.Maps;
*/
public class MutationState implements SQLCloseable {
private static final Logger logger = LoggerFactory.getLogger(MutationState.class);
-
+ private static final TransactionCodec CODEC = new TransactionCodec();
+
private PhoenixConnection connection;
private final long maxSize;
- private final ImmutableBytesPtr tempPtr = new ImmutableBytesPtr();
private final Map<TableRef, Map<ImmutableBytesPtr,RowMutationState>> mutations;
+ private final List<TransactionAware> txAwares;
+ private final TransactionContext txContext;
+ private final Set<String> uncommittedPhysicalNames = Sets.newHashSetWithExpectedSize(10);
+
+ private Transaction tx;
private long sizeOffset;
private int numRows = 0;
+ private boolean txStarted = false;
+
private final MutationMetricQueue mutationMetricQueue;
private ReadMetricQueue readMetricQueue;
- MutationState(long maxSize, PhoenixConnection connection,
- Map<TableRef, Map<ImmutableBytesPtr, RowMutationState>> mutations) {
- this.maxSize = maxSize;
- this.connection = connection;
- this.mutations = mutations;
- boolean isMetricsEnabled = connection.isRequestLevelMetricsEnabled();
- this.mutationMetricQueue = isMetricsEnabled ? new MutationMetricQueue()
- : NoOpMutationMetricsQueue.NO_OP_MUTATION_METRICS_QUEUE;
- }
-
public MutationState(long maxSize, PhoenixConnection connection) {
- this(maxSize,connection,0);
+ this(maxSize,connection, null);
+ }
+
+ public MutationState(MutationState mutationState) {
+ this(mutationState.maxSize, mutationState.connection, mutationState.getTransaction());
}
public MutationState(long maxSize, PhoenixConnection connection, long sizeOffset) {
- this(maxSize, connection, Maps.<TableRef, Map<ImmutableBytesPtr,RowMutationState>>newHashMapWithExpectedSize(connection.getMutateBatchSize()));
+ this(maxSize, connection, null, sizeOffset);
+ }
+
+ private MutationState(long maxSize, PhoenixConnection connection, Transaction tx) {
+ this(maxSize,connection, tx, 0);
+ }
+
+ private MutationState(long maxSize, PhoenixConnection connection, Transaction tx, long sizeOffset) {
+ this(maxSize, connection, Maps.<TableRef, Map<ImmutableBytesPtr,RowMutationState>>newHashMapWithExpectedSize(connection.getMutateBatchSize()), tx);
this.sizeOffset = sizeOffset;
}
+ MutationState(long maxSize, PhoenixConnection connection,
+ Map<TableRef, Map<ImmutableBytesPtr, RowMutationState>> mutations,
+ Transaction tx) {
+ this.maxSize = maxSize;
+ this.connection = connection;
+ this.mutations = mutations;
+ boolean isMetricsEnabled = connection.isRequestLevelMetricsEnabled();
+ this.mutationMetricQueue = isMetricsEnabled ? new MutationMetricQueue()
+ : NoOpMutationMetricsQueue.NO_OP_MUTATION_METRICS_QUEUE;
+ this.tx = tx;
+ if (tx == null) {
+ this.txAwares = Collections.emptyList();
+ TransactionSystemClient txServiceClient = this.connection
+ .getQueryServices().getTransactionSystemClient();
+ this.txContext = new TransactionContext(txServiceClient);
+ } else {
+ txAwares = Lists.newArrayList();
+ txContext = null;
+ }
+ }
+
public MutationState(TableRef table, Map<ImmutableBytesPtr,RowMutationState> mutations, long sizeOffset, long maxSize, PhoenixConnection connection) {
- this(maxSize, connection, sizeOffset);
+ this(maxSize, connection, null, sizeOffset);
this.mutations.put(table, mutations);
this.numRows = mutations.size();
+ this.tx = connection.getMutationState().getTransaction();
throwIfTooBig();
}
+ public boolean checkpointIfNeccessary(MutationPlan plan) throws SQLException {
+ Transaction currentTx = getTransaction();
+ if (getTransaction() == null || plan.getTargetRef() == null || plan.getTargetRef().getTable() == null || !plan.getTargetRef().getTable().isTransactional()) {
+ return false;
+ }
+ Set<TableRef> sources = plan.getSourceRefs();
+ if (sources.isEmpty()) {
+ return false;
+ }
+ // For a DELETE statement, we're always querying the table being deleted from. This isn't
+ // a problem, but it potentially could be if there are other references to the same table
+ // nested in the DELETE statement (as a sub query or join, for example).
+ TableRef ignoreForExcludeCurrent = plan.getOperation() == Operation.DELETE && sources.size() == 1 ? plan.getTargetRef() : null;
+ boolean excludeCurrent = false;
+ String targetPhysicalName = plan.getTargetRef().getTable().getPhysicalName().getString();
+ for (TableRef source : sources) {
+ if (source.getTable().isTransactional() && !source.equals(ignoreForExcludeCurrent)) {
+ String sourcePhysicalName = source.getTable().getPhysicalName().getString();
+ if (targetPhysicalName.equals(sourcePhysicalName)) {
+ excludeCurrent = true;
+ break;
+ }
+ }
+ }
+ // If we're querying the same table we're updating, we must exclude our writes to
+ // it from being visible.
+ if (excludeCurrent) {
+ // If any source tables have uncommitted data prior to last checkpoint,
+ // then we must create a new checkpoint.
+ boolean hasUncommittedData = false;
+ for (TableRef source : sources) {
+ String sourcePhysicalName = source.getTable().getPhysicalName().getString();
+ if (source.getTable().isTransactional() && uncommittedPhysicalNames.contains(sourcePhysicalName)) {
+ hasUncommittedData = true;
+ break;
+ }
+ }
+ if (hasUncommittedData) {
+ try {
+ if (txContext == null) {
+ currentTx = tx = connection.getQueryServices().getTransactionSystemClient().checkpoint(currentTx);
+ } else {
+ txContext.checkpoint();
+ currentTx = tx = txContext.getCurrentTransaction();
+ }
+ // Since we've checkpointed, we can clear out uncommitted set, since a statement run afterwards
+ // should see all this data.
+ uncommittedPhysicalNames.clear();
+ } catch (TransactionFailureException e) {
+ throw new SQLException(e);
+ }
+ }
+ // Since we're querying our own table while mutating it, we must exclude
+ // see our current mutations, otherwise we can get erroneous results (for DELETE)
+ // or get into an infinite loop (for UPSERT SELECT).
+ currentTx.setVisibility(VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
+ return true;
+ }
+ return false;
+ }
+
+ private void addTransactionParticipant(TransactionAware txAware) throws SQLException {
+ if (txContext == null) {
+ txAwares.add(txAware);
+ assert(tx != null);
+ txAware.startTx(tx);
+ } else {
+ txContext.addTransactionAware(txAware);
+ }
+ }
+
+ // Though MutationState is not thread safe in general, this method should be because it may
+ // be called by TableResultIterator in a multi-threaded manner. Since we do not want to expose
+ // the Transaction outside of MutationState, this seems reasonable, as the member variables
+ // would not change as these threads are running.
+ public HTableInterface getHTable(PTable table) throws SQLException {
+ HTableInterface htable = this.getConnection().getQueryServices().getTable(table.getPhysicalName().getBytes());
+ Transaction currentTx;
+ if (table.isTransactional() && (currentTx=getTransaction()) != null) {
+ TransactionAwareHTable txAware = TransactionUtil.getTransactionAwareHTable(htable, table);
+ // Using cloned mutationState as we may have started a new transaction already
+ // if auto commit is true and we need to use the original one here.
+ txAware.startTx(currentTx);
+ htable = txAware;
+ }
+ return htable;
+ }
+
+ public PhoenixConnection getConnection() {
+ return connection;
+ }
+
+ // Kept private as the Transaction may change when check pointed. Keeping it private ensures
+ // no one holds on to a stale copy.
+ private Transaction getTransaction() {
+ return tx != null ? tx : txContext != null ? txContext.getCurrentTransaction() : null;
+ }
+
+ public boolean isTransactionStarted() {
+ return getTransaction() != null;
+ }
+
+ public long getReadPointer() {
+ Transaction tx = getTransaction();
+ return tx == null ? HConstants.LATEST_TIMESTAMP : tx.getReadPointer();
+ }
+
+ // For testing
+ public long getWritePointer() {
+ Transaction tx = getTransaction();
+ return tx == null ? HConstants.LATEST_TIMESTAMP : tx.getWritePointer();
+ }
+
+ // For testing
+ public VisibilityLevel getVisibilityLevel() {
+ Transaction tx = getTransaction();
+ return tx == null ? null : tx.getVisibilityLevel();
+ }
+
+ public boolean startTransaction() throws SQLException {
+ if (txContext == null) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.NULL_TRANSACTION_CONTEXT).build().buildException();
+ }
+
+ if (connection.getSCN() != null) {
+ throw new SQLExceptionInfo.Builder(
+ SQLExceptionCode.CANNOT_START_TRANSACTION_WITH_SCN_SET)
+ .build().buildException();
+ }
+
+ try {
+ if (!txStarted) {
+ txContext.start();
+ txStarted = true;
+ return true;
+ }
+ } catch (TransactionFailureException e) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.TRANSACTION_FAILED).setRootCause(e).build().buildException();
+ }
+ return false;
+ }
+
public static MutationState emptyMutationState(long maxSize, PhoenixConnection connection) {
- MutationState state = new MutationState(maxSize, connection, Collections.<TableRef, Map<ImmutableBytesPtr,RowMutationState>>emptyMap());
+ MutationState state = new MutationState(maxSize, connection, Collections.<TableRef, Map<ImmutableBytesPtr,RowMutationState>>emptyMap(), null);
state.sizeOffset = 0;
return state;
}
@@ -153,6 +344,13 @@ public class MutationState implements SQLCloseable {
if (this == newMutationState) { // Doesn't make sense
return;
}
+ if (txContext != null) {
+ for (TransactionAware txAware : newMutationState.txAwares) {
+ txContext.addTransactionAware(txAware);
+ }
+ } else {
+ txAwares.addAll(newMutationState.txAwares);
+ }
this.sizeOffset += newMutationState.sizeOffset;
// Merge newMutation with this one, keeping state from newMutation for any overlaps
for (Map.Entry<TableRef, Map<ImmutableBytesPtr,RowMutationState>> entry : newMutationState.mutations.entrySet()) {
@@ -202,7 +400,8 @@ public class MutationState implements SQLCloseable {
throwIfTooBig();
}
- private static ImmutableBytesPtr getNewRowKeyWithRowTimestamp(ImmutableBytesPtr ptr, long rowTimestamp, PTable table) {
+
+ private static ImmutableBytesPtr getNewRowKeyWithRowTimestamp(ImmutableBytesPtr ptr, long rowTimestamp, PTable table) {
RowKeySchema schema = table.getRowKeySchema();
int rowTimestampColPos = table.getRowTimestampColPos();
Field rowTimestampField = schema.getField(rowTimestampColPos);
@@ -223,51 +422,15 @@ public class MutationState implements SQLCloseable {
return ptr;
}
- private Iterator<Pair<byte[],List<Mutation>>> addRowMutations(final TableRef tableRef, final Map<ImmutableBytesPtr, RowMutationState> values, long timestamp, boolean includeMutableIndexes) {
+ private Iterator<Pair<byte[],List<Mutation>>> addRowMutations(final TableRef tableRef, final Map<ImmutableBytesPtr, RowMutationState> values, final long timestamp, boolean includeMutableIndexes, final boolean sendAll) {
final PTable table = tableRef.getTable();
- boolean tableWithRowTimestampCol = table.getRowTimestampColPos() != -1;
final Iterator<PTable> indexes = // Only maintain tables with immutable rows through this client-side mechanism
(table.isImmutableRows() || includeMutableIndexes) ?
IndexMaintainer.nonDisabledIndexIterator(table.getIndexes().iterator()) :
Iterators.<PTable>emptyIterator();
- final List<Mutation> mutations = Lists.newArrayListWithExpectedSize(values.size());
+ final List<Mutation> mutationList = Lists.newArrayListWithExpectedSize(values.size());
final List<Mutation> mutationsPertainingToIndex = indexes.hasNext() ? Lists.<Mutation>newArrayListWithExpectedSize(values.size()) : null;
- Iterator<Map.Entry<ImmutableBytesPtr,RowMutationState>> iterator = values.entrySet().iterator();
- long timestampToUse = timestamp;
- while (iterator.hasNext()) {
- Map.Entry<ImmutableBytesPtr,RowMutationState> rowEntry = iterator.next();
- ImmutableBytesPtr key = rowEntry.getKey();
- RowMutationState state = rowEntry.getValue();
- if (tableWithRowTimestampCol) {
- RowTimestampColInfo rowTsColInfo = state.getRowTimestampColInfo();
- if (rowTsColInfo.useServerTimestamp()) {
- // regenerate the key with this timestamp.
- key = getNewRowKeyWithRowTimestamp(key, timestampToUse, table);
- } else {
- if (rowTsColInfo.getTimestamp() != null) {
- timestampToUse = rowTsColInfo.getTimestamp();
- }
- }
- }
- PRow row = table.newRow(connection.getKeyValueBuilder(), timestampToUse, key);
- List<Mutation> rowMutations, rowMutationsPertainingToIndex;
- if (rowEntry.getValue().getColumnValues() == PRow.DELETE_MARKER) { // means delete
- row.delete();
- rowMutations = row.toRowMutations();
- // Row deletes for index tables are processed by running a re-written query
- // against the index table (as this allows for flexibility in being able to
- // delete rows).
- rowMutationsPertainingToIndex = Collections.emptyList();
- } else {
- for (Map.Entry<PColumn,byte[]> valueEntry : rowEntry.getValue().getColumnValues().entrySet()) {
- row.setValue(valueEntry.getKey(), valueEntry.getValue());
- }
- rowMutations = row.toRowMutations();
- rowMutationsPertainingToIndex = rowMutations;
- }
- mutations.addAll(rowMutations);
- if (mutationsPertainingToIndex != null) mutationsPertainingToIndex.addAll(rowMutationsPertainingToIndex);
- }
+ generateMutations(tableRef, timestamp, values, mutationList, mutationsPertainingToIndex);
return new Iterator<Pair<byte[],List<Mutation>>>() {
boolean isFirst = true;
@@ -280,14 +443,24 @@ public class MutationState implements SQLCloseable {
public Pair<byte[], List<Mutation>> next() {
if (isFirst) {
isFirst = false;
- return new Pair<byte[],List<Mutation>>(table.getPhysicalName().getBytes(),mutations);
+ return new Pair<byte[],List<Mutation>>(table.getPhysicalName().getBytes(), mutationList);
}
PTable index = indexes.next();
List<Mutation> indexMutations;
try {
indexMutations =
IndexUtil.generateIndexData(table, index, mutationsPertainingToIndex,
- tempPtr, connection.getKeyValueBuilder(), connection);
+ connection.getKeyValueBuilder(), connection);
+ // we may also have to include delete mutations for immutable tables if we are not processing all the tables in the mutations map
+ if (!sendAll) {
+ TableRef key = new TableRef(index);
+ Map<ImmutableBytesPtr, RowMutationState> rowToColumnMap = mutations.remove(key);
+ if (rowToColumnMap!=null) {
+ final List<Mutation> deleteMutations = Lists.newArrayList();
+ generateMutations(tableRef, timestamp, rowToColumnMap, deleteMutations, null);
+ indexMutations.addAll(deleteMutations);
+ }
+ }
} catch (SQLException e) {
throw new IllegalDataException(e);
}
@@ -301,28 +474,84 @@ public class MutationState implements SQLCloseable {
};
}
+
+ private void generateMutations(final TableRef tableRef, long timestamp,
+ final Map<ImmutableBytesPtr, RowMutationState> values,
+ final List<Mutation> mutationList, final List<Mutation> mutationsPertainingToIndex) {
+ final PTable table = tableRef.getTable();
+ boolean tableWithRowTimestampCol = table.getRowTimestampColPos() != -1;
+ Iterator<Map.Entry<ImmutableBytesPtr, RowMutationState>> iterator =
+ values.entrySet().iterator();
+ long timestampToUse = timestamp;
+ while (iterator.hasNext()) {
+ Map.Entry<ImmutableBytesPtr, RowMutationState> rowEntry = iterator.next();
+ ImmutableBytesPtr key = rowEntry.getKey();
+ RowMutationState state = rowEntry.getValue();
+ if (tableWithRowTimestampCol) {
+ RowTimestampColInfo rowTsColInfo = state.getRowTimestampColInfo();
+ if (rowTsColInfo.useServerTimestamp()) {
+ // regenerate the key with this timestamp.
+ key = getNewRowKeyWithRowTimestamp(key, timestampToUse, table);
+ } else {
+ if (rowTsColInfo.getTimestamp() != null) {
+ timestampToUse = rowTsColInfo.getTimestamp();
+ }
+ }
+ }
+ PRow row =
+ tableRef.getTable()
+ .newRow(connection.getKeyValueBuilder(), timestampToUse, key);
+ List<Mutation> rowMutations, rowMutationsPertainingToIndex;
+ if (rowEntry.getValue().getColumnValues() == PRow.DELETE_MARKER) { // means delete
+ row.delete();
+ rowMutations = row.toRowMutations();
+ // Row deletes for index tables are processed by running a re-written query
+ // against the index table (as this allows for flexibility in being able to
+ // delete rows).
+ rowMutationsPertainingToIndex = Collections.emptyList();
+ } else {
+ for (Map.Entry<PColumn, byte[]> valueEntry : rowEntry.getValue().getColumnValues()
+ .entrySet()) {
+ row.setValue(valueEntry.getKey(), valueEntry.getValue());
+ }
+ rowMutations = row.toRowMutations();
+ rowMutationsPertainingToIndex = rowMutations;
+ }
+ mutationList.addAll(rowMutations);
+ if (mutationsPertainingToIndex != null) mutationsPertainingToIndex
+ .addAll(rowMutationsPertainingToIndex);
+ }
+ }
/**
* Get the unsorted list of HBase mutations for the tables with uncommitted data.
* @return list of HBase mutations for uncommitted data.
*/
+ public Iterator<Pair<byte[],List<Mutation>>> toMutations(Long timestamp) {
+ return toMutations(false, timestamp);
+ }
+
public Iterator<Pair<byte[],List<Mutation>>> toMutations() {
- return toMutations(false);
+ return toMutations(false, null);
}
public Iterator<Pair<byte[],List<Mutation>>> toMutations(final boolean includeMutableIndexes) {
+ return toMutations(includeMutableIndexes, null);
+ }
+
+ public Iterator<Pair<byte[],List<Mutation>>> toMutations(final boolean includeMutableIndexes, final Long tableTimestamp) {
final Iterator<Map.Entry<TableRef, Map<ImmutableBytesPtr,RowMutationState>>> iterator = this.mutations.entrySet().iterator();
if (!iterator.hasNext()) {
return Iterators.emptyIterator();
}
Long scn = connection.getSCN();
- final long timestamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
+ final long timestamp = (tableTimestamp!=null && tableTimestamp!=QueryConstants.UNSET_TIMESTAMP) ? tableTimestamp : (scn == null ? HConstants.LATEST_TIMESTAMP : scn);
return new Iterator<Pair<byte[],List<Mutation>>>() {
private Map.Entry<TableRef, Map<ImmutableBytesPtr,RowMutationState>> current = iterator.next();
private Iterator<Pair<byte[],List<Mutation>>> innerIterator = init();
private Iterator<Pair<byte[],List<Mutation>>> init() {
- return addRowMutations(current.getKey(), current.getValue(), timestamp, includeMutableIndexes);
+ return addRowMutations(current.getKey(), current.getValue(), timestamp, includeMutableIndexes, true);
}
@Override
@@ -346,58 +575,62 @@ public class MutationState implements SQLCloseable {
};
}
- /**
- * Validates that the meta data is valid against the server meta data if we haven't yet done so.
- * Otherwise, for every UPSERT VALUES call, we'd need to hit the server to see if the meta data
- * has changed.
- * @param connection
- * @return the server time to use for the upsert
- * @throws SQLException if the table or any columns no longer exist
- */
- private long[] validate() throws SQLException {
- int i = 0;
- Long scn = connection.getSCN();
- MetaDataClient client = new MetaDataClient(connection);
- long[] timeStamps = new long[this.mutations.size()];
- for (Map.Entry<TableRef, Map<ImmutableBytesPtr,RowMutationState>> entry : mutations.entrySet()) {
- TableRef tableRef = entry.getKey();
- long serverTimeStamp = tableRef.getTimeStamp();
- PTable table = tableRef.getTable();
- // If we're auto committing, we've already validated the schema when we got the ColumnResolver,
- // so no need to do it again here.
- if (!connection.getAutoCommit()) {
- MetaDataMutationResult result = client.updateCache(table.getSchemaName().getString(), table.getTableName().getString());
- long timestamp = result.getMutationTime();
- if (timestamp != QueryConstants.UNSET_TIMESTAMP) {
- serverTimeStamp = timestamp;
- if (result.wasUpdated()) {
- // TODO: use bitset?
- table = result.getTable();
- PColumn[] columns = new PColumn[table.getColumns().size()];
- for (Map.Entry<ImmutableBytesPtr,RowMutationState> rowEntry : entry.getValue().entrySet()) {
- RowMutationState valueEntry = rowEntry.getValue();
- if (valueEntry != null) {
- Map<PColumn, byte[]> colValues = valueEntry.getColumnValues();
- if (colValues != PRow.DELETE_MARKER) {
- for (PColumn column : colValues.keySet()) {
- columns[column.getPosition()] = column;
- }
- }
- }
+ /**
+ * Validates that the meta data is valid against the server meta data if we haven't yet done so.
+ * Otherwise, for every UPSERT VALUES call, we'd need to hit the server to see if the meta data
+ * has changed.
+ * @param connection
+ * @return the server time to use for the upsert
+ * @throws SQLException if the table or any columns no longer exist
+ */
+ private long[] validateAll() throws SQLException {
+ int i = 0;
+ long[] timeStamps = new long[this.mutations.size()];
+ for (Map.Entry<TableRef, Map<ImmutableBytesPtr,RowMutationState>> entry : mutations.entrySet()) {
+ TableRef tableRef = entry.getKey();
+ timeStamps[i++] = validate(tableRef, entry.getValue());
+ }
+ return timeStamps;
+ }
+
+ private long validate(TableRef tableRef, Map<ImmutableBytesPtr, RowMutationState> rowKeyToColumnMap) throws SQLException {
+ Long scn = connection.getSCN();
+ MetaDataClient client = new MetaDataClient(connection);
+ long serverTimeStamp = tableRef.getTimeStamp();
+ PTable table = tableRef.getTable();
+ // If we're auto committing, we've already validated the schema when we got the ColumnResolver,
+ // so no need to do it again here.
+ if (!connection.getAutoCommit()) {
+ MetaDataMutationResult result = client.updateCache(table.getSchemaName().getString(), table.getTableName().getString());
+ long timestamp = result.getMutationTime();
+ if (timestamp != QueryConstants.UNSET_TIMESTAMP) {
+ serverTimeStamp = timestamp;
+ if (result.wasUpdated()) {
+ // TODO: use bitset?
+ table = result.getTable();
+ PColumn[] columns = new PColumn[table.getColumns().size()];
+ for (Map.Entry<ImmutableBytesPtr,RowMutationState> rowEntry : rowKeyToColumnMap.entrySet()) {
+ RowMutationState valueEntry = rowEntry.getValue();
+ if (valueEntry != null) {
+ Map<PColumn, byte[]> colValues = valueEntry.getColumnValues();
+ if (colValues != PRow.DELETE_MARKER) {
+ for (PColumn column : colValues.keySet()) {
+ columns[column.getPosition()] = column;
+ }
+ }
}
- for (PColumn column : columns) {
- if (column != null) {
- table.getColumnFamily(column.getFamilyName().getString()).getColumn(column.getName().getString());
- }
+ }
+ for (PColumn column : columns) {
+ if (column != null) {
+ table.getColumnFamily(column.getFamilyName().getString()).getColumn(column.getName().getString());
}
- tableRef.setTable(table);
}
+ tableRef.setTable(table);
}
}
- timeStamps[i++] = scn == null ? serverTimeStamp == QueryConstants.UNSET_TIMESTAMP ? HConstants.LATEST_TIMESTAMP : serverTimeStamp : scn;
}
- return timeStamps;
- }
+ return scn == null ? serverTimeStamp == QueryConstants.UNSET_TIMESTAMP ? HConstants.LATEST_TIMESTAMP : serverTimeStamp : scn;
+ }
private static long calculateMutationSize(List<Mutation> mutations) {
long byteSize = 0;
@@ -410,149 +643,292 @@ public class MutationState implements SQLCloseable {
return byteSize;
}
+ private boolean hasKeyValueColumn(PTable table, PTable index) {
+ IndexMaintainer maintainer = index.getIndexMaintainer(table, connection);
+ return !maintainer.getAllColumns().isEmpty();
+ }
+
+ private void divideImmutableIndexes(Iterator<PTable> enabledImmutableIndexes, PTable table, List<PTable> rowKeyIndexes, List<PTable> keyValueIndexes) {
+ while (enabledImmutableIndexes.hasNext()) {
+ PTable index = enabledImmutableIndexes.next();
+ if (index.getIndexType() != IndexType.LOCAL) {
+ if (hasKeyValueColumn(table, index)) {
+ keyValueIndexes.add(index);
+ } else {
+ rowKeyIndexes.add(index);
+ }
+ }
+ }
+ }
+ private class MetaDataAwareHTable extends DelegateHTable {
+ private final TableRef tableRef;
+
+ private MetaDataAwareHTable(HTableInterface delegate, TableRef tableRef) {
+ super(delegate);
+ this.tableRef = tableRef;
+ }
+
+ /**
+ * Called by Tephra when a transaction is aborted. We have this wrapper so that we get an
+ * opportunity to attach our index meta data to the mutations such that we can also undo
+ * the index mutations.
+ */
+ @Override
+ public void delete(List<Delete> deletes) throws IOException {
+ try {
+ PTable table = tableRef.getTable();
+ List<PTable> indexes = table.getIndexes();
+ Iterator<PTable> enabledIndexes = IndexMaintainer.nonDisabledIndexIterator(indexes.iterator());
+ if (enabledIndexes.hasNext()) {
+ List<PTable> keyValueIndexes = Collections.emptyList();
+ ImmutableBytesWritable indexMetaDataPtr = new ImmutableBytesWritable();
+ boolean attachMetaData = table.getIndexMaintainers(indexMetaDataPtr, connection);
+ if (table.isImmutableRows()) {
+ List<PTable> rowKeyIndexes = Lists.newArrayListWithExpectedSize(indexes.size());
+ keyValueIndexes = Lists.newArrayListWithExpectedSize(indexes.size());
+ divideImmutableIndexes(enabledIndexes, table, rowKeyIndexes, keyValueIndexes);
+ // Generate index deletes for immutable indexes that only reference row key
+ // columns and submit directly here.
+ ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+ for (PTable index : rowKeyIndexes) {
+ List<Delete> indexDeletes = IndexUtil.generateDeleteIndexData(table, index, deletes, ptr, connection.getKeyValueBuilder(), connection);
+ HTableInterface hindex = connection.getQueryServices().getTable(index.getPhysicalName().getBytes());
+ hindex.delete(indexDeletes);
+ }
+ }
+
+ // If we have mutable indexes, local immutable indexes, or global immutable indexes
+ // that reference key value columns, setup index meta data and attach here. In this
+ // case updates to the indexes will be generated on the server side.
+ // An alternative would be to let Tephra track the row keys for the immutable index
+ // by adding it as a transaction participant (soon we can prevent any conflict
+ // detection from occurring) with the downside being the additional memory required.
+ if (!keyValueIndexes.isEmpty()) {
+ attachMetaData = true;
+ IndexMaintainer.serializeAdditional(table, indexMetaDataPtr, keyValueIndexes, connection);
+ }
+ if (attachMetaData) {
+ setMetaDataOnMutations(tableRef, deletes, indexMetaDataPtr);
+ }
+ }
+ delegate.delete(deletes);
+ } catch (SQLException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
@SuppressWarnings("deprecation")
- public void commit() throws SQLException {
+ private void send(Iterator<TableRef> tableRefIterator) throws SQLException {
int i = 0;
- PName tenantId = connection.getTenantId();
- long[] serverTimeStamps = validate();
- Iterator<Map.Entry<TableRef, Map<ImmutableBytesPtr,RowMutationState>>> iterator = this.mutations.entrySet().iterator();
+ long[] serverTimeStamps = null;
+ boolean sendAll = false;
+ if (tableRefIterator == null) {
+ serverTimeStamps = validateAll();
+ tableRefIterator = mutations.keySet().iterator();
+ sendAll = true;
+ }
+
// add tracing for this operation
try (TraceScope trace = Tracing.startNewSpan(connection, "Committing mutations to tables")) {
Span span = trace.getSpan();
- while (iterator.hasNext()) {
- Map.Entry<TableRef, Map<ImmutableBytesPtr,RowMutationState>> entry = iterator.next();
- // at this point we are going through mutations for each table
-
- Map<ImmutableBytesPtr,RowMutationState> valuesMap = entry.getValue();
- // above is mutations for a table where the first part is the row key and the second part is column values.
-
- TableRef tableRef = entry.getKey();
- PTable table = tableRef.getTable();
- table.getIndexMaintainers(tempPtr, connection);
- boolean hasIndexMaintainers = tempPtr.getLength() > 0;
- boolean isDataTable = true;
- long serverTimestamp = serverTimeStamps[i++];
- Iterator<Pair<byte[],List<Mutation>>> mutationsIterator = addRowMutations(tableRef, valuesMap, serverTimestamp, false);
- // above returns an iterator of pair where the first
- while (mutationsIterator.hasNext()) {
- Pair<byte[],List<Mutation>> pair = mutationsIterator.next();
- byte[] htableName = pair.getFirst();
- List<Mutation> mutations = pair.getSecond();
-
- //create a span per target table
- //TODO maybe we can be smarter about the table name to string here?
- Span child = Tracing.child(span,"Writing mutation batch for table: "+Bytes.toString(htableName));
-
- int retryCount = 0;
- boolean shouldRetry = false;
- do {
- ServerCache cache = null;
- if (hasIndexMaintainers && isDataTable) {
- byte[] attribValue = null;
- byte[] uuidValue;
- if (IndexMetaDataCacheClient.useIndexMetadataCache(connection, mutations, tempPtr.getLength())) {
- IndexMetaDataCacheClient client = new IndexMetaDataCacheClient(connection, tableRef);
- cache = client.addIndexMetadataCache(mutations, tempPtr);
- child.addTimelineAnnotation("Updated index metadata cache");
- uuidValue = cache.getId();
- // If we haven't retried yet, retry for this case only, as it's possible that
- // a split will occur after we send the index metadata cache to all known
- // region servers.
- shouldRetry = true;
- } else {
- attribValue = ByteUtil.copyKeyBytesIfNecessary(tempPtr);
- uuidValue = ServerCacheClient.generateId();
- }
- // Either set the UUID to be able to access the index metadata from the cache
- // or set the index metadata directly on the Mutation
- for (Mutation mutation : mutations) {
- if (tenantId != null) {
- byte[] tenantIdBytes = ScanUtil.getTenantIdBytes(
- table.getRowKeySchema(),
- table.getBucketNum()!=null,
- tenantId);
- mutation.setAttribute(PhoenixRuntime.TENANT_ID_ATTRIB, tenantIdBytes);
- }
- mutation.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
- if (attribValue != null) {
- mutation.setAttribute(PhoenixIndexCodec.INDEX_MD, attribValue);
- }
- }
- }
-
- SQLException sqlE = null;
- HTableInterface hTable = connection.getQueryServices().getTable(htableName);
- try {
- long numMutations = mutations.size();
+ ImmutableBytesWritable indexMetaDataPtr = new ImmutableBytesWritable();
+ while (tableRefIterator.hasNext()) {
+ // at this point we are going through mutations for each table
+ TableRef tableRef = tableRefIterator.next();
+ Map<ImmutableBytesPtr, RowMutationState> valuesMap = mutations.get(tableRef);
+ if (valuesMap == null || valuesMap.isEmpty()) {
+ continue;
+ }
+ PTable table = tableRef.getTable();
+ // Track tables to which we've sent uncommitted data
+ if (table.isTransactional()) {
+ uncommittedPhysicalNames.add(table.getPhysicalName().getString());
+ }
+ table.getIndexMaintainers(indexMetaDataPtr, connection);
+ boolean isDataTable = true;
+ // Validate as we go if transactional since we can undo if a problem occurs (which is unlikely)
+ long serverTimestamp = serverTimeStamps == null ? validate(tableRef, valuesMap) : serverTimeStamps[i++];
+ Iterator<Pair<byte[],List<Mutation>>> mutationsIterator = addRowMutations(tableRef, valuesMap, serverTimestamp, false, sendAll);
+ while (mutationsIterator.hasNext()) {
+ Pair<byte[],List<Mutation>> pair = mutationsIterator.next();
+ byte[] htableName = pair.getFirst();
+ List<Mutation> mutationList = pair.getSecond();
+
+ //create a span per target table
+ //TODO maybe we can be smarter about the table name to string here?
+ Span child = Tracing.child(span,"Writing mutation batch for table: "+Bytes.toString(htableName));
+
+ int retryCount = 0;
+ boolean shouldRetry = false;
+ do {
+ ServerCache cache = null;
+ if (isDataTable) {
+ cache = setMetaDataOnMutations(tableRef, mutationList, indexMetaDataPtr);
+ }
+
+ // If we haven't retried yet, retry for this case only, as it's possible that
+ // a split will occur after we send the index metadata cache to all known
+ // region servers.
+ shouldRetry = cache != null;
+ SQLException sqlE = null;
+ HTableInterface hTable = connection.getQueryServices().getTable(htableName);
+ try {
+ if (table.isTransactional()) {
+ // If we have indexes, wrap the HTable in a delegate HTable that
+ // will attach the necessary index meta data in the event of a
+ // rollback
+ if (!table.getIndexes().isEmpty()) {
+ hTable = new MetaDataAwareHTable(hTable, tableRef);
+ }
+ TransactionAwareHTable txnAware = TransactionUtil.getTransactionAwareHTable(hTable, table);
+ // Don't add immutable indexes (those are the only ones that would participate
+ // during a commit), as we don't need conflict detection for these.
+ if (isDataTable) {
+ // Even for immutable, we need to do this so that an abort has the state
+ // necessary to generate the rows to delete.
+ addTransactionParticipant(txnAware);
+ } else {
+ txnAware.startTx(getTransaction());
+ }
+ hTable = txnAware;
+ }
+ long numMutations = mutationList.size();
GLOBAL_MUTATION_BATCH_SIZE.update(numMutations);
long startTime = System.currentTimeMillis();
- child.addTimelineAnnotation("Attempt " + retryCount);
- hTable.batch(mutations);
- child.stop();
+ child.addTimelineAnnotation("Attempt " + retryCount);;
+ hTable.batch(mutationList);
+ child.stop();
+ child.stop();
shouldRetry = false;
long mutationCommitTime = System.currentTimeMillis() - startTime;
GLOBAL_MUTATION_COMMIT_TIME.update(mutationCommitTime);
- long mutationSizeBytes = calculateMutationSize(mutations);
+ long mutationSizeBytes = calculateMutationSize(mutationList);
MutationMetric mutationsMetric = new MutationMetric(numMutations, mutationSizeBytes, mutationCommitTime);
mutationMetricQueue.addMetricsForTable(Bytes.toString(htableName), mutationsMetric);
- } catch (Exception e) {
- SQLException inferredE = ServerUtil.parseServerExceptionOrNull(e);
- if (inferredE != null) {
- if (shouldRetry && retryCount == 0 && inferredE.getErrorCode() == SQLExceptionCode.INDEX_METADATA_NOT_FOUND.getErrorCode()) {
- // Swallow this exception once, as it's possible that we split after sending the index metadata
- // and one of the region servers doesn't have it. This will cause it to have it the next go around.
- // If it fails again, we don't retry.
- String msg = "Swallowing exception and retrying after clearing meta cache on connection. " + inferredE;
- logger.warn(LogUtil.addCustomAnnotations(msg, connection));
- connection.getQueryServices().clearTableRegionCache(htableName);
+ } catch (Exception e) {
+ SQLException inferredE = ServerUtil.parseServerExceptionOrNull(e);
+ if (inferredE != null) {
+ if (shouldRetry && retryCount == 0 && inferredE.getErrorCode() == SQLExceptionCode.INDEX_METADATA_NOT_FOUND.getErrorCode()) {
+ // Swallow this exception once, as it's possible that we split after sending the index metadata
+ // and one of the region servers doesn't have it. This will cause it to have it the next go around.
+ // If it fails again, we don't retry.
+ String msg = "Swallowing exception and retrying after clearing meta cache on connection. " + inferredE;
+ logger.warn(LogUtil.addCustomAnnotations(msg, connection));
+ connection.getQueryServices().clearTableRegionCache(htableName);
+
+ // add a new child span as this one failed
+ child.addTimelineAnnotation(msg);
+ child.stop();
+ child = Tracing.child(span,"Failed batch, attempting retry");
+
+ continue;
+ }
+ e = inferredE;
+ }
+ // Throw to client with both what was committed so far and what is left to be committed.
+ // That way, client can either undo what was done or try again with what was not done.
+ sqlE = new CommitException(e, getUncommittedStatementIndexes());
+ } finally {
+ try {
+ if (cache != null) {
+ cache.close();
+ }
+ } finally {
+ try {
+ hTable.close();
+ }
+ catch (IOException e) {
+ if (sqlE != null) {
+ sqlE.setNextException(ServerUtil.parseServerException(e));
+ } else {
+ sqlE = ServerUtil.parseServerException(e);
+ }
+ }
+ if (sqlE != null) {
+ // clear pending mutations
+ mutations.clear();
+ throw sqlE;
+ }
+ }
+ }
+ } while (shouldRetry && retryCount++ < 1);
+ isDataTable = false;
+ }
+ if (tableRef.getTable().getType() != PTableType.INDEX) {
+ numRows -= valuesMap.size();
+ }
+ // Remove batches as we process them
+ if (sendAll) {
+ tableRefIterator.remove(); // Iterating through actual map in this case
+ } else {
+ mutations.remove(tableRef);
+ }
+ }
+ }
+ // Note that we cannot assume that *all* mutations have been sent, since we've optimized this
+ // now to only send the mutations for the tables we're querying, hence we've removed the
+ // assertions that we're here before.
+ }
- // add a new child span as this one failed
- child.addTimelineAnnotation(msg);
- child.stop();
- child = Tracing.child(span,"Failed batch, attempting retry");
+ public byte[] encodeTransaction() throws SQLException {
+ try {
+ return CODEC.encode(getTransaction());
+ } catch (IOException e) {
+ throw new SQLException(e);
+ }
+ }
+
+ public static Transaction decodeTransaction(byte[] txnBytes) throws IOException {
+ return (txnBytes == null || txnBytes.length==0) ? null : CODEC.decode(txnBytes);
+ }
- continue;
- }
- e = inferredE;
- }
- sqlE = new CommitException(e, getUncommittedStatementIndexes());
- } finally {
- try {
- hTable.close();
- } catch (IOException e) {
- if (sqlE != null) {
- sqlE.setNextException(ServerUtil.parseServerException(e));
- } else {
- sqlE = ServerUtil.parseServerException(e);
- }
- } finally {
- try {
- if (cache != null) {
- cache.close();
- }
- } finally {
- if (sqlE != null) {
- throw sqlE;
- }
- }
- }
- }
- } while (shouldRetry && retryCount++ < 1);
- isDataTable = false;
- }
- if (tableRef.getTable().getType() != PTableType.INDEX) {
- numRows -= entry.getValue().size();
+ private ServerCache setMetaDataOnMutations(TableRef tableRef, List<? extends Mutation> mutations,
+ ImmutableBytesWritable indexMetaDataPtr) throws SQLException {
+ PTable table = tableRef.getTable();
+ byte[] tenantId = connection.getTenantId() == null ? null : connection.getTenantId().getBytes();
+ ServerCache cache = null;
+ byte[] attribValue = null;
+ byte[] uuidValue = null;
+ byte[] txState = ByteUtil.EMPTY_BYTE_ARRAY;
+ if (table.isTransactional()) {
+ txState = encodeTransaction();
+ }
+ boolean hasIndexMetaData = indexMetaDataPtr.getLength() > 0;
+ if (hasIndexMetaData) {
+ if (IndexMetaDataCacheClient.useIndexMetadataCache(connection, mutations, indexMetaDataPtr.getLength() + txState.length)) {
+ IndexMetaDataCacheClient client = new IndexMetaDataCacheClient(connection, tableRef);
+ cache = client.addIndexMetadataCache(mutations, indexMetaDataPtr, txState);
+ uuidValue = cache.getId();
+ } else {
+ attribValue = ByteUtil.copyKeyBytesIfNecessary(indexMetaDataPtr);
+ uuidValue = ServerCacheClient.generateId();
+ }
+ } else if (txState.length == 0) {
+ return null;
+ }
+ // Either set the UUID to be able to access the index metadata from the cache
+ // or set the index metadata directly on the Mutation
+ for (Mutation mutation : mutations) {
+ if (tenantId != null) {
+ mutation.setAttribute(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
+ }
+ mutation.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
+ if (attribValue != null) {
+ mutation.setAttribute(PhoenixIndexCodec.INDEX_MD, attribValue);
+ if (txState.length > 0) {
+ mutation.setAttribute(BaseScannerRegionObserver.TX_STATE, txState);
}
- iterator.remove(); // Remove batches as we process them
+ } else if (!hasIndexMetaData && txState.length > 0) {
+ mutation.setAttribute(BaseScannerRegionObserver.TX_STATE, txState);
}
}
- assert(numRows==0);
- assert(this.mutations.isEmpty());
+ return cache;
}
- public void rollback(PhoenixConnection connection) throws SQLException {
+ private void clear() throws SQLException {
this.mutations.clear();
numRows = 0;
}
@@ -570,6 +946,100 @@ public class MutationState implements SQLCloseable {
@Override
public void close() throws SQLException {
}
+
+ private void reset() {
+ txStarted = false;
+ tx = null;
+ uncommittedPhysicalNames.clear();
+ }
+
+ public void rollback() throws SQLException {
+ clear();
+ txAwares.clear();
+ if (txContext != null) {
+ try {
+ if (txStarted) {
+ txContext.abort();
+ }
+ } catch (TransactionFailureException e) {
+ throw new SQLException(e); // TODO: error code
+ } finally {
+ reset();
+ }
+ }
+ }
+
+ public void commit() throws SQLException {
+ boolean sendMutationsFailed=false;
+ try {
+ send();
+ } catch (Throwable t) {
+ sendMutationsFailed=true;
+ throw t;
+ } finally {
+ txAwares.clear();
+ if (txContext != null) {
+ try {
+ if (txStarted && !sendMutationsFailed) {
+ txContext.finish();
+ }
+ } catch (TransactionFailureException e) {
+ try {
+ txContext.abort(e);
+ // abort and throw the original commit failure exception
+ throw TransactionUtil.getTransactionFailureException(e);
+ } catch (TransactionFailureException e1) {
+ // if abort fails and throw the abort failure exception
+ throw TransactionUtil.getTransactionFailureException(e1);
+ }
+ } finally {
+ if (!sendMutationsFailed) {
+ reset();
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Support read-your-own-write semantics by sending uncommitted data to HBase prior to running a
+ * query. In this way, they are visible to subsequent reads but are not actually committed until
+ * commit is called.
+ * @param tableRefs
+ * @return true if at least partially transactional and false otherwise.
+ * @throws SQLException
+ */
+ public boolean sendUncommitted(Iterator<TableRef> tableRefs) throws SQLException {
+ Transaction currentTx = getTransaction();
+ if (currentTx != null) {
+ // Initialize visibility so that transactions see their own writes.
+ // The checkpoint() method will set it to not see writes if necessary.
+ currentTx.setVisibility(VisibilityLevel.SNAPSHOT);
+ }
+ Iterator<TableRef> filteredTableRefs = Iterators.filter(tableRefs, new Predicate<TableRef>(){
+ @Override
+ public boolean apply(TableRef tableRef) {
+ return tableRef.getTable().isTransactional();
+ }
+ });
+ if (filteredTableRefs.hasNext()) {
+ // FIXME: strip table alias to prevent equality check from failing due to alias mismatch on null alias.
+ // We really should be keying the tables based on the physical table name.
+ List<TableRef> strippedAliases = Lists.newArrayListWithExpectedSize(mutations.keySet().size());
+ while (filteredTableRefs.hasNext()) {
+ TableRef tableRef = filteredTableRefs.next();
+ strippedAliases.add(new TableRef(null, tableRef.getTable(), tableRef.getTimeStamp(), tableRef.getLowerBoundTimeStamp(), tableRef.hasDynamicCols()));
+ }
+ startTransaction();
+ send(strippedAliases.iterator());
+ return true;
+ }
+ return false;
+ }
+
+ public void send() throws SQLException {
+ send(null);
+ }
public static int[] joinSortedIntArrays(int[] a, int[] b) {
int[] result = new int[a.length + b.length];
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
index 3098980..19b3e6b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
@@ -151,7 +151,7 @@ public class ScanPlan extends BaseQueryPlan {
return spoolingResultIteratorFactory;
} else {
return new ChunkedResultIterator.ChunkedResultIteratorFactory(
- spoolingResultIteratorFactory, table);
+ spoolingResultIteratorFactory, context.getConnection().getMutationState(), table);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
index dea82a8..ab7534a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
@@ -89,6 +89,8 @@ import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TrustedByteArrayOutputStream;
+import co.cask.tephra.TxConstants;
+
import com.google.common.base.Predicate;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
@@ -874,7 +876,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> {
}
else if (kv.getTypeByte() == KeyValue.Type.DeleteFamily.getCode()
// Since we don't include the index rows in the change set for txn tables, we need to detect row deletes that have transformed by TransactionProcessor
- ) {
+ || (CellUtil.matchingQualifier(kv, TxConstants.FAMILY_DELETE_QUALIFIER) && CellUtil.matchingValue(kv, HConstants.EMPTY_BYTE_ARRAY))) {
nDeleteCF++;
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java
index 70ddc86..05a01b9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java
@@ -24,7 +24,6 @@ import java.util.List;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
import org.apache.phoenix.cache.ServerCacheClient;
import org.apache.phoenix.cache.ServerCacheClient.ServerCache;
import org.apache.phoenix.compile.ScanRanges;
@@ -59,7 +58,7 @@ public class IndexMetaDataCacheClient {
* @param mutations the list of mutations that will be sent in a batch to server
* @param indexMetaDataByteLength length in bytes of the index metadata cache
*/
- public static boolean useIndexMetadataCache(PhoenixConnection connection, List<Mutation> mutations, int indexMetaDataByteLength) {
+ public static boolean useIndexMetadataCache(PhoenixConnection connection, List<? extends Mutation> mutations, int indexMetaDataByteLength) {
ReadOnlyProps props = connection.getQueryServices().getProps();
int threshold = props.getInt(INDEX_MUTATE_BATCH_SIZE_THRESHOLD_ATTRIB, QueryServicesOptions.DEFAULT_INDEX_MUTATE_BATCH_SIZE_THRESHOLD);
return (indexMetaDataByteLength > ServerCacheClient.UUID_LENGTH && mutations.size() > threshold);
@@ -72,25 +71,26 @@ public class IndexMetaDataCacheClient {
* @throws MaxServerCacheSizeExceededException if size of hash cache exceeds max allowed
* size
*/
- public ServerCache addIndexMetadataCache(List<Mutation> mutations, ImmutableBytesWritable ptr) throws SQLException {
+ public ServerCache addIndexMetadataCache(List<? extends Mutation> mutations, ImmutableBytesWritable ptr, byte[] txState) throws SQLException {
/**
* Serialize and compress hashCacheTable
*/
- return serverCache.addServerCache(ScanUtil.newScanRanges(mutations), ptr, new IndexMetaDataCacheFactory(), cacheUsingTableRef);
+ return serverCache.addServerCache(ScanUtil.newScanRanges(mutations), ptr, txState, new IndexMetaDataCacheFactory(), cacheUsingTableRef);
}
/**
* Send the index metadata cahce to all region servers for regions that will handle the mutations.
+ * @param txState TODO
* @return client-side {@link ServerCache} representing the added index metadata cache
* @throws SQLException
* @throws MaxServerCacheSizeExceededException if size of hash cache exceeds max allowed
* size
*/
- public ServerCache addIndexMetadataCache(ScanRanges ranges, ImmutableBytesWritable ptr) throws SQLException {
+ public ServerCache addIndexMetadataCache(ScanRanges ranges, ImmutableBytesWritable ptr, byte[] txState) throws SQLException {
/**
* Serialize and compress hashCacheTable
*/
- return serverCache.addServerCache(ranges, ptr, new IndexMetaDataCacheFactory(), cacheUsingTableRef);
+ return serverCache.addServerCache(ranges, ptr, txState, new IndexMetaDataCacheFactory(), cacheUsingTableRef);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheFactory.java
index 488db44..7ced2f8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheFactory.java
@@ -24,11 +24,15 @@ import java.io.IOException;
import java.sql.SQLException;
import java.util.List;
+import co.cask.tephra.Transaction;
+
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.cache.IndexMetaDataCache;
import org.apache.phoenix.coprocessor.ServerCachingProtocol.ServerCacheFactory;
+import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
import org.apache.phoenix.memory.MemoryManager.MemoryChunk;
+import org.apache.phoenix.util.TransactionUtil;
public class IndexMetaDataCacheFactory implements ServerCacheFactory {
public IndexMetaDataCacheFactory() {
@@ -43,10 +47,16 @@ public class IndexMetaDataCacheFactory implements ServerCacheFactory {
}
@Override
- public Closeable newCache (ImmutableBytesWritable cachePtr, final MemoryChunk chunk) throws SQLException {
+ public Closeable newCache (ImmutableBytesWritable cachePtr, byte[] txState, final MemoryChunk chunk) throws SQLException {
// just use the standard keyvalue builder - this doesn't really need to be fast
- final List<IndexMaintainer> maintainers =
+ final List<IndexMaintainer> maintainers =
IndexMaintainer.deserialize(cachePtr, GenericKeyValueBuilder.INSTANCE);
+ final Transaction txn;
+ try {
+ txn = txState.length!=0 ? MutationState.decodeTransaction(txState) : null;
+ } catch (IOException e) {
+ throw new SQLException(e);
+ }
return new IndexMetaDataCache() {
@Override
@@ -58,6 +68,11 @@ public class IndexMetaDataCacheFactory implements ServerCacheFactory {
public List<IndexMaintainer> getIndexMaintainers() {
return maintainers;
}
+
+ @Override
+ public Transaction getTransaction() {
+ return txn;
+ }
};
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
index 2fd168a..806a20a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
@@ -75,7 +75,7 @@ import com.google.common.collect.Multimap;
*
* @since 2.1
*/
-public class PhoenixIndexFailurePolicy extends KillServerOnFailurePolicy {
+public class PhoenixIndexFailurePolicy extends KillServerOnFailurePolicy {
private static final Log LOG = LogFactory.getLog(PhoenixIndexFailurePolicy.class);
private RegionCoprocessorEnvironment env;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaData.java
index 3dcc44e..60ae915 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaData.java
@@ -22,6 +22,8 @@ import java.sql.SQLException;
import java.util.List;
import java.util.Map;
+import co.cask.tephra.Transaction;
+
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.cache.GlobalCache;
@@ -46,8 +48,10 @@ public class PhoenixIndexMetaData implements IndexMetaData {
byte[] uuid = attributes.get(PhoenixIndexCodec.INDEX_UUID);
if (uuid == null) { return IndexMetaDataCache.EMPTY_INDEX_META_DATA_CACHE; }
byte[] md = attributes.get(PhoenixIndexCodec.INDEX_MD);
+ byte[] txState = attributes.get(BaseScannerRegionObserver.TX_STATE);
if (md != null) {
final List<IndexMaintainer> indexMaintainers = IndexMaintainer.deserialize(md);
+ final Transaction txn = MutationState.decodeTransaction(txState);
return new IndexMetaDataCache() {
@Override
@@ -58,6 +62,11 @@ public class PhoenixIndexMetaData implements IndexMetaData {
return indexMaintainers;
}
+ @Override
+ public Transaction getTransaction() {
+ return txn;
+ }
+
};
} else {
byte[] tenantIdBytes = attributes.get(PhoenixRuntime.TENANT_ID_ATTRIB);
@@ -80,6 +89,10 @@ public class PhoenixIndexMetaData implements IndexMetaData {
this.attributes = attributes;
}
+ public Transaction getTransaction() {
+ return indexMetaDataCache.getTransaction();
+ }
+
public List<IndexMaintainer> getIndexMaintainers() {
return indexMetaDataCache.getIndexMaintainers();
}
[2/9] phoenix git commit: PHOENIX-1674 Snapshot isolation transaction
support through Tephra (James Taylor, Thomas D'Silva)
Posted by td...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 21548b0..cb9b831 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -72,6 +72,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
@@ -169,6 +170,7 @@ import org.apache.phoenix.parse.SQLParser;
import org.apache.phoenix.parse.SelectStatement;
import org.apache.phoenix.parse.TableName;
import org.apache.phoenix.parse.UpdateStatisticsStatement;
+import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.ConnectionQueryServices.Feature;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
@@ -193,10 +195,13 @@ import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.StringUtil;
+import org.apache.phoenix.util.TransactionUtil;
import org.apache.phoenix.util.UpgradeUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import co.cask.tephra.TxConstants;
+
import com.google.common.base.Objects;
import com.google.common.collect.Iterators;
import com.google.common.collect.ListMultimap;
@@ -230,8 +235,9 @@ public class MetaDataClient {
VIEW_INDEX_ID + "," +
INDEX_TYPE + "," +
STORE_NULLS + "," +
- BASE_COLUMN_COUNT +
- ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+ BASE_COLUMN_COUNT + "," +
+ TRANSACTIONAL +
+ ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
private static final String CREATE_LINK =
"UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " +
TENANT_ID + "," +
@@ -372,7 +378,7 @@ public class MetaDataClient {
MetaDataMutationResult result = updateCache(schemaName, tableName, true);
return result.getMutationTime();
}
-
+
/**
* Update the cache with the latest as of the connection scn.
* @param schemaName
@@ -391,6 +397,10 @@ public class MetaDataClient {
public MetaDataMutationResult updateCache(PName tenantId, String schemaName, String tableName) throws SQLException {
return updateCache(tenantId, schemaName, tableName, false);
}
+
+ public MetaDataMutationResult updateCache(PName tenantId, String schemaName, String tableName, boolean alwaysHitServer) throws SQLException {
+ return updateCache(tenantId, schemaName, tableName, alwaysHitServer, null);
+ }
/**
* Update the cache with the latest as of the connection scn.
@@ -415,24 +425,47 @@ public class MetaDataClient {
long clientTimeStamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
return clientTimeStamp;
}
-
+
+ private long getCurrentScn() {
+ Long scn = connection.getSCN();
+ long currentScn = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
+ return currentScn;
+ }
+
private MetaDataMutationResult updateCache(PName tenantId, String schemaName, String tableName,
- boolean alwaysHitServer) throws SQLException { // TODO: pass byte[] herez
- long clientTimeStamp = getClientTimeStamp();
+ boolean alwaysHitServer, Long resolvedTimestamp) throws SQLException { // TODO: pass byte[] herez
boolean systemTable = SYSTEM_CATALOG_SCHEMA.equals(schemaName);
// System tables must always have a null tenantId
tenantId = systemTable ? null : tenantId;
PTable table = null;
+ PTableRef tableRef = null;
String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
long tableTimestamp = HConstants.LATEST_TIMESTAMP;
+ long tableResolvedTimestamp = HConstants.LATEST_TIMESTAMP;
try {
- table = connection.getTable(new PTableKey(tenantId, fullTableName));
+ tableRef = connection.getTableRef(new PTableKey(tenantId, fullTableName));
+ table = tableRef.getTable();
tableTimestamp = table.getTimeStamp();
+ tableResolvedTimestamp = tableRef.getResolvedTimeStamp();
} catch (TableNotFoundException e) {
}
- // Don't bother with server call: we can't possibly find a newer table
- if (table != null && !alwaysHitServer && (systemTable || tableTimestamp == clientTimeStamp - 1)) {
- return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS,QueryConstants.UNSET_TIMESTAMP,table);
+
+ boolean defaultTransactional = connection.getQueryServices().getProps().getBoolean(
+ QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB,
+ QueryServicesOptions.DEFAULT_TRANSACTIONAL);
+ // start a txn if all table are transactional by default or if we found the table in the cache and it is transactional
+ // TODO if system tables become transactional remove the check
+ boolean isTransactional = defaultTransactional || (table!=null && table.isTransactional());
+ if (!systemTable && isTransactional && !connection.getMutationState().isTransactionStarted()) {
+ connection.getMutationState().startTransaction();
+ }
+ resolvedTimestamp = resolvedTimestamp==null ? TransactionUtil.getResolvedTimestamp(connection, isTransactional, HConstants.LATEST_TIMESTAMP) : resolvedTimestamp;
+ // Do not make rpc to getTable if
+ // 1. table is a system table
+ // 2. table was already resolved as of that timestamp
+ if (table != null && !alwaysHitServer
+ && (systemTable || resolvedTimestamp == tableResolvedTimestamp)) {
+ return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, QueryConstants.UNSET_TIMESTAMP, table);
}
int maxTryCount = tenantId == null ? 1 : 2;
@@ -442,7 +475,12 @@ public class MetaDataClient {
do {
final byte[] schemaBytes = PVarchar.INSTANCE.toBytes(schemaName);
final byte[] tableBytes = PVarchar.INSTANCE.toBytes(tableName);
- result = connection.getQueryServices().getTable(tenantId, schemaBytes, tableBytes, tableTimestamp, clientTimeStamp);
+ ConnectionQueryServices queryServices = connection.getQueryServices();
+ result = queryServices.getTable(tenantId, schemaBytes, tableBytes, tableTimestamp, resolvedTimestamp);
+ // if the table was assumed to be transactional, but is actually not transactional then re-resolve as of the right timestamp (and vice versa)
+ if (table==null && result.getTable()!=null && result.getTable().isTransactional()!=isTransactional) {
+ result = queryServices.getTable(tenantId, schemaBytes, tableBytes, tableTimestamp, TransactionUtil.getResolvedTimestamp(connection, result.getTable().isTransactional(), HConstants.LATEST_TIMESTAMP));
+ }
if (SYSTEM_CATALOG_SCHEMA.equals(schemaName)) {
return result;
@@ -466,15 +504,20 @@ public class MetaDataClient {
// timestamp, we can handle this such that we don't ask the
// server again.
if (table != null) {
+ // Ensures that table in result is set to table found in our cache.
if (code == MutationCode.TABLE_ALREADY_EXISTS) {
- // Ensures that table in result is set to table found in our cache.
- result.setTable(table);
- // Although this table is up-to-date, the parent table may not be.
- // In this case, we update the parent table which may in turn pull
- // in indexes to add to this table.
- if (addIndexesFromPhysicalTable(result)) {
- connection.addTable(result.getTable());
- }
+ result.setTable(table);
+ // Although this table is up-to-date, the parent table may not be.
+ // In this case, we update the parent table which may in turn pull
+ // in indexes to add to this table.
+ long resolvedTime = TransactionUtil.getResolvedTime(connection, result);
+ if (addIndexesFromPhysicalTable(result, resolvedTimestamp)) {
+ connection.addTable(result.getTable(), resolvedTime);
+ }
+ else {
+ // if we aren't adding the table, we still need to update the resolved time of the table
+ connection.updateResolvedTimestamp(table, resolvedTime);
+ }
return result;
}
// If table was not found at the current time stamp and we have one cached, remove it.
@@ -566,10 +609,11 @@ public class MetaDataClient {
* of the table for which we just updated.
* TODO: combine this round trip with the one that updates the cache for the child table.
* @param result the result from updating the cache for the current table.
+ * @param resolvedTimestamp timestamp at which child table was resolved
* @return true if the PTable contained by result was modified and false otherwise
* @throws SQLException if the physical table cannot be found
*/
- private boolean addIndexesFromPhysicalTable(MetaDataMutationResult result) throws SQLException {
+ private boolean addIndexesFromPhysicalTable(MetaDataMutationResult result, Long resolvedTimestamp) throws SQLException {
PTable table = result.getTable();
// If not a view or if a view directly over an HBase table, there's nothing to do
if (table.getType() != PTableType.VIEW || table.getViewType() == ViewType.MAPPED) {
@@ -578,7 +622,7 @@ public class MetaDataClient {
String physicalName = table.getPhysicalName().getString();
String schemaName = SchemaUtil.getSchemaNameFromFullName(physicalName);
String tableName = SchemaUtil.getTableNameFromFullName(physicalName);
- MetaDataMutationResult parentResult = updateCache(null, schemaName, tableName, false);
+ MetaDataMutationResult parentResult = updateCache(null, schemaName, tableName, false, resolvedTimestamp);
PTable physicalTable = parentResult.getTable();
if (physicalTable == null) {
throw new TableNotFoundException(schemaName, tableName);
@@ -782,7 +826,7 @@ public class MetaDataClient {
public MutationState createTable(CreateTableStatement statement, byte[][] splits, PTable parent, String viewStatement, ViewType viewType, byte[][] viewColumnConstants, BitSet isViewColumnReferenced) throws SQLException {
PTable table = createTableInternal(statement, splits, parent, viewStatement, viewType, viewColumnConstants, isViewColumnReferenced, null, null);
- if (table == null || table.getType() == PTableType.VIEW) {
+ if (table == null || table.getType() == PTableType.VIEW || table.isTransactional()) {
return new MutationState(0,connection);
}
// Hack to get around the case when an SCN is specified on the connection.
@@ -1137,7 +1181,7 @@ public class MetaDataClient {
if (hbaseVersion < PhoenixDatabaseMetaData.MUTABLE_SI_VERSION_THRESHOLD) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_MUTABLE_INDEXES).setTableName(indexTableName.getTableName()).build().buildException();
}
- if (connection.getQueryServices().hasInvalidIndexConfiguration()) {
+ if (!connection.getQueryServices().isMutableIndexWALCodecInstalled() && !dataTable.isTransactional()) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_MUTABLE_INDEX_CONFIG).setTableName(indexTableName.getTableName()).build().buildException();
}
}
@@ -1268,19 +1312,19 @@ public class MetaDataClient {
// as there's no need to burn another sequence value.
if (allocateIndexId && indexId == null) {
Long scn = connection.getSCN();
- long timestamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
PName tenantId = connection.getTenantId();
String tenantIdStr = tenantId == null ? null : connection.getTenantId().getString();
PName physicalName = dataTable.getPhysicalName();
int nSequenceSaltBuckets = connection.getQueryServices().getSequenceSaltBuckets();
SequenceKey key = MetaDataUtil.getViewIndexSequenceKey(tenantIdStr, physicalName, nSequenceSaltBuckets);
- // Create at parent timestamp as we know that will be earlier than now
- // and earlier than any SCN if one is set.
+ // if scn is set create at scn-1, so we can see the sequence or else use latest timestamp (so that latest server time is used)
+ long sequenceTimestamp = scn!=null ? scn-1 : HConstants.LATEST_TIMESTAMP;
createSequence(key.getTenantId(), key.getSchemaName(), key.getSequenceName(),
true, Short.MIN_VALUE, 1, 1, false, Long.MIN_VALUE, Long.MAX_VALUE,
- dataTable.getTimeStamp());
+ sequenceTimestamp);
long[] seqValues = new long[1];
SQLException[] sqlExceptions = new SQLException[1];
+ long timestamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
connection.getQueryServices().incrementSequences(Collections.singletonList(new SequenceAllocation(key, 1)),
Math.max(timestamp, dataTable.getTimeStamp()), seqValues, sqlExceptions);
if (sqlExceptions[0] != null) {
@@ -1397,7 +1441,7 @@ public class MetaDataClient {
functionUpsert.setString(5, function.getJarPath());
functionUpsert.setString(6, function.getReturnType());
functionUpsert.execute();
- functionData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ functionData.addAll(connection.getMutationState().toMutations(null).next().getSecond());
connection.rollback();
MetaDataMutationResult result = connection.getQueryServices().createFunction(functionData, function, stmt.isTemporary());
MutationCode code = result.getMutationCode();
@@ -1498,29 +1542,36 @@ public class MetaDataClient {
long clientTimeStamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
boolean multiTenant = false;
boolean storeNulls = false;
+ boolean transactional = false;
Integer saltBucketNum = null;
String defaultFamilyName = null;
boolean isImmutableRows = false;
List<PName> physicalNames = Collections.emptyList();
boolean addSaltColumn = false;
boolean rowKeyOrderOptimizable = true;
+ Long timestamp = null;
if (parent != null && tableType == PTableType.INDEX) {
- // Index on view
- // TODO: Can we support a multi-tenant index directly on a multi-tenant
- // table instead of only a view? We don't have anywhere to put the link
- // from the table to the index, though.
- if (indexType == IndexType.LOCAL || (parent.getType() == PTableType.VIEW && parent.getViewType() != ViewType.MAPPED)) {
- PName physicalName = parent.getPhysicalName();
- saltBucketNum = parent.getBucketNum();
- addSaltColumn = (saltBucketNum != null && indexType != IndexType.LOCAL);
- defaultFamilyName = parent.getDefaultFamilyName() == null ? null : parent.getDefaultFamilyName().getString();
- if (indexType == IndexType.LOCAL) {
- saltBucketNum = null;
- // Set physical name of local index table
- physicalNames = Collections.singletonList(PNameFactory.newName(MetaDataUtil.getLocalIndexPhysicalName(physicalName.getBytes())));
- } else {
- // Set physical name of view index table
- physicalNames = Collections.singletonList(PNameFactory.newName(MetaDataUtil.getViewIndexPhysicalName(physicalName.getBytes())));
+ transactional = parent.isTransactional();
+ timestamp = TransactionUtil.getTableTimestamp(connection, transactional);
+ storeNulls = parent.getStoreNulls();
+ if (tableType == PTableType.INDEX) {
+ // Index on view
+ // TODO: Can we support a multi-tenant index directly on a multi-tenant
+ // table instead of only a view? We don't have anywhere to put the link
+ // from the table to the index, though.
+ if (indexType == IndexType.LOCAL || (parent.getType() == PTableType.VIEW && parent.getViewType() != ViewType.MAPPED)) {
+ PName physicalName = parent.getPhysicalName();
+ saltBucketNum = parent.getBucketNum();
+ addSaltColumn = (saltBucketNum != null && indexType != IndexType.LOCAL);
+ defaultFamilyName = parent.getDefaultFamilyName() == null ? null : parent.getDefaultFamilyName().getString();
+ if (indexType == IndexType.LOCAL) {
+ saltBucketNum = null;
+ // Set physical name of local index table
+ physicalNames = Collections.singletonList(PNameFactory.newName(MetaDataUtil.getLocalIndexPhysicalName(physicalName.getBytes())));
+ } else {
+ // Set physical name of view index table
+ physicalNames = Collections.singletonList(PNameFactory.newName(MetaDataUtil.getViewIndexPhysicalName(physicalName.getBytes())));
+ }
}
}
@@ -1536,7 +1587,7 @@ public class MetaDataClient {
incrementStatement.execute();
// Get list of mutations and add to table meta data that will be passed to server
// to guarantee order. This row will always end up last
- tableMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ tableMetaData.addAll(connection.getMutationState().toMutations(timestamp).next().getSecond());
connection.rollback();
// Add row linking from data table row to index table row
@@ -1561,12 +1612,10 @@ public class MetaDataClient {
}
Map<String,Object> tableProps = Maps.newHashMapWithExpectedSize(statement.getProps().size());
- Map<String,Object> commonFamilyProps = Collections.emptyMap();
+ Map<String,Object> commonFamilyProps = Maps.newHashMapWithExpectedSize(statement.getProps().size() + 1);
// Somewhat hacky way of determining if property is for HColumnDescriptor or HTableDescriptor
HColumnDescriptor defaultDescriptor = new HColumnDescriptor(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES);
if (!statement.getProps().isEmpty()) {
- commonFamilyProps = Maps.newHashMapWithExpectedSize(statement.getProps().size());
-
Collection<Pair<String,Object>> props = statement.getProps().get(QueryConstants.ALL_FAMILY_PROPERTIES_KEY);
for (Pair<String,Object> prop : props) {
if (defaultDescriptor.getValue(prop.getFirst()) == null) {
@@ -1580,7 +1629,7 @@ public class MetaDataClient {
// Although unusual, it's possible to set a mapped VIEW as having immutable rows.
// This tells Phoenix that you're managing the index maintenance yourself.
if (tableType != PTableType.INDEX && (tableType != PTableType.VIEW || viewType == ViewType.MAPPED)) {
- Boolean isImmutableRowsProp = (Boolean) tableProps.remove(PTable.IS_IMMUTABLE_ROWS_PROP_NAME);
+ Boolean isImmutableRowsProp = (Boolean) tableProps.get(PTable.IS_IMMUTABLE_ROWS_PROP_NAME);
if (isImmutableRowsProp == null) {
isImmutableRows = connection.getQueryServices().getProps().getBoolean(QueryServices.IMMUTABLE_ROWS_ATTRIB, QueryServicesOptions.DEFAULT_IMMUTABLE_ROWS);
} else {
@@ -1590,7 +1639,7 @@ public class MetaDataClient {
// Can't set any of these on views or shared indexes on views
if (tableType != PTableType.VIEW && indexId == null) {
- saltBucketNum = (Integer) tableProps.remove(PhoenixDatabaseMetaData.SALT_BUCKETS);
+ saltBucketNum = (Integer) tableProps.get(PhoenixDatabaseMetaData.SALT_BUCKETS);
if (saltBucketNum != null) {
if (saltBucketNum < 0 || saltBucketNum > SaltingUtil.MAX_BUCKET_NUM) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_BUCKET_NUM).build().buildException();
@@ -1607,42 +1656,110 @@ public class MetaDataClient {
addSaltColumn = (saltBucketNum != null);
}
- boolean removedProp = false;
// Can't set MULTI_TENANT or DEFAULT_COLUMN_FAMILY_NAME on an INDEX or a non mapped VIEW
if (tableType != PTableType.INDEX && (tableType != PTableType.VIEW || viewType == ViewType.MAPPED)) {
- Boolean multiTenantProp = (Boolean) tableProps.remove(PhoenixDatabaseMetaData.MULTI_TENANT);
+ Boolean multiTenantProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.MULTI_TENANT);
multiTenant = Boolean.TRUE.equals(multiTenantProp);
- defaultFamilyName = (String)tableProps.remove(PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME);
- removedProp = (defaultFamilyName != null);
+ defaultFamilyName = (String)tableProps.get(PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME);
}
boolean disableWAL = false;
- Boolean disableWALProp = (Boolean) tableProps.remove(PhoenixDatabaseMetaData.DISABLE_WAL);
+ Boolean disableWALProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.DISABLE_WAL);
if (disableWALProp != null) {
disableWAL = disableWALProp;
}
- Boolean storeNullsProp = (Boolean) tableProps.remove(PhoenixDatabaseMetaData.STORE_NULLS);
- storeNulls = storeNullsProp == null
- ? connection.getQueryServices().getProps().getBoolean(
- QueryServices.DEFAULT_STORE_NULLS_ATTRIB,
- QueryServicesOptions.DEFAULT_STORE_NULLS)
- : storeNullsProp;
+ Boolean storeNullsProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.STORE_NULLS);
+ if (storeNullsProp == null) {
+ if (parent == null) {
+ storeNulls = connection.getQueryServices().getProps().getBoolean(
+ QueryServices.DEFAULT_STORE_NULLS_ATTRIB,
+ QueryServicesOptions.DEFAULT_STORE_NULLS);
+ tableProps.put(PhoenixDatabaseMetaData.STORE_NULLS, Boolean.valueOf(storeNulls));
+ }
+ } else {
+ storeNulls = storeNullsProp;
+ }
+ Boolean transactionalProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.TRANSACTIONAL);
+ if (transactionalProp != null && parent != null) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.ONLY_TABLE_MAY_BE_DECLARED_TRANSACTIONAL)
+ .setSchemaName(schemaName).setTableName(tableName)
+ .build().buildException();
+ }
+ if (parent == null) {
+ if (transactionalProp == null) {
+ transactional = connection.getQueryServices().getProps().getBoolean(
+ QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB,
+ QueryServicesOptions.DEFAULT_TRANSACTIONAL);
+ } else {
+ transactional = transactionalProp;
+ }
+ }
+ tableProps.put(PhoenixDatabaseMetaData.TRANSACTIONAL, transactional);
+ if (transactional) {
+ // If TTL set, use Tephra TTL property name instead
+ Object ttl = commonFamilyProps.remove(HColumnDescriptor.TTL);
+ if (ttl != null) {
+ commonFamilyProps.put(TxConstants.PROPERTY_TTL, ttl);
+ }
+ }
+
+ boolean sharedTable = statement.getTableType() == PTableType.VIEW || indexId != null;
+ if (transactional) {
+ // Tephra uses an empty value cell as its delete marker, so we need to turn on
+ // storeNulls for transactional tables.
+ // If we use regular column delete markers (which is what non transactional tables
+ // use), then they get converted
+ // on the server, but this can mess up our secondary index code as the changes get
+ // committed prior to the
+ // maintenance code being able to see the prior state to update the rows correctly.
+ if (Boolean.FALSE.equals(storeNullsProp)) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.STORE_NULLS_MUST_BE_TRUE_FOR_TRANSACTIONAL)
+ .setSchemaName(schemaName).setTableName(tableName)
+ .build().buildException();
+ }
+ // Force STORE_NULLS to true when transactional as Tephra cannot deal with column deletes
+ storeNulls = true;
+ tableProps.put(PhoenixDatabaseMetaData.STORE_NULLS, Boolean.TRUE);
+
+ if (!sharedTable) {
+ Integer maxVersionsProp = (Integer) commonFamilyProps.get(HConstants.VERSIONS);
+ if (maxVersionsProp == null) {
+ if (parent != null) {
+ HTableDescriptor desc = connection.getQueryServices().getTableDescriptor(parent.getPhysicalName().getBytes());
+ if (desc != null) {
+ maxVersionsProp = desc.getFamily(SchemaUtil.getEmptyColumnFamily(parent)).getMaxVersions();
+ }
+ }
+ if (maxVersionsProp == null) {
+ maxVersionsProp = connection.getQueryServices().getProps().getInt(
+ QueryServices.MAX_VERSIONS_TRANSACTIONAL_ATTRIB,
+ QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL);
+ }
+ commonFamilyProps.put(HConstants.VERSIONS, maxVersionsProp);
+ }
+ }
+ }
+ timestamp = timestamp==null ? TransactionUtil.getTableTimestamp(connection, transactional) : timestamp;
// Delay this check as it is supported to have IMMUTABLE_ROWS and SALT_BUCKETS defined on views
- if ((statement.getTableType() == PTableType.VIEW || indexId != null) && !tableProps.isEmpty()) {
- throw new SQLExceptionInfo.Builder(SQLExceptionCode.VIEW_WITH_PROPERTIES).build()
- .buildException();
- }
- if (removedProp) {
- tableProps.put(PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME, defaultFamilyName);
+ if (sharedTable) {
+ if (tableProps.get(PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME) != null) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.DEFAULT_COLUMN_FAMILY_ON_SHARED_TABLE)
+ .setSchemaName(schemaName).setTableName(tableName)
+ .build().buildException();
+ }
+ if (SchemaUtil.hasHTableDescriptorProps(tableProps)) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.VIEW_WITH_PROPERTIES).build()
+ .buildException();
+ }
}
List<ColumnDef> colDefs = statement.getColumnDefs();
List<PColumn> columns;
LinkedHashSet<PColumn> pkColumns;
- if (tenantId != null && (tableType != PTableType.VIEW && indexId == null)) {
+ if (tenantId != null && !sharedTable) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_CREATE_TENANT_SPECIFIC_TABLE)
.setSchemaName(schemaName).setTableName(tableName).build().buildException();
}
@@ -1866,8 +1983,8 @@ public class MetaDataClient {
Collections.<PTable>emptyList(), isImmutableRows,
Collections.<PName>emptyList(), defaultFamilyName == null ? null :
PNameFactory.newName(defaultFamilyName), null,
- Boolean.TRUE.equals(disableWAL), false, false, null, indexId, indexType, true);
- connection.addTable(table);
+ Boolean.TRUE.equals(disableWAL), false, false, null, indexId, indexType, true, false);
+ connection.addTable(table, MetaDataProtocol.MIN_TABLE_TIMESTAMP);
} else if (tableType == PTableType.INDEX && indexId == null) {
if (tableProps.get(HTableDescriptor.MAX_FILESIZE) == null) {
int nIndexRowKeyColumns = isPK ? 1 : pkColumnsNames.size();
@@ -1922,7 +2039,7 @@ public class MetaDataClient {
addColumnMutation(schemaName, tableName, column, colUpsert, parentTableName, pkName, keySeq, saltBucketNum != null);
}
- tableMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ tableMetaData.addAll(connection.getMutationState().toMutations(timestamp).next().getSecond());
connection.rollback();
String dataTableName = parent == null || tableType == PTableType.VIEW ? null : parent.getTableName().getString();
@@ -1968,9 +2085,10 @@ public class MetaDataClient {
} else {
tableUpsert.setInt(20, BASE_TABLE_BASE_COLUMN_COUNT);
}
+ tableUpsert.setBoolean(21, transactional);
tableUpsert.execute();
- tableMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ tableMetaData.addAll(connection.getMutationState().toMutations(timestamp).next().getSecond());
connection.rollback();
/*
@@ -1996,7 +2114,9 @@ public class MetaDataClient {
MutationCode code = result.getMutationCode();
switch(code) {
case TABLE_ALREADY_EXISTS:
- addTableToCache(result);
+ if (result.getTable() != null) { // Can happen for transactional table that already exists as HBase table
+ addTableToCache(result);
+ }
if (!statement.ifNotExists()) {
throw new TableAlreadyExistsException(schemaName, tableName, result.getTable());
}
@@ -2016,11 +2136,11 @@ public class MetaDataClient {
default:
PName newSchemaName = PNameFactory.newName(schemaName);
PTable table = PTableImpl.makePTable(
- tenantId, newSchemaName, PNameFactory.newName(tableName), tableType, indexState, result.getMutationTime(),
+ tenantId, newSchemaName, PNameFactory.newName(tableName), tableType, indexState, timestamp!=null ? timestamp : result.getMutationTime(),
PTable.INITIAL_SEQ_NUM, pkName == null ? null : PNameFactory.newName(pkName), saltBucketNum, columns,
dataTableName == null ? null : newSchemaName, dataTableName == null ? null : PNameFactory.newName(dataTableName), Collections.<PTable>emptyList(), isImmutableRows,
physicalNames, defaultFamilyName == null ? null : PNameFactory.newName(defaultFamilyName), viewStatement, Boolean.TRUE.equals(disableWAL), multiTenant, storeNulls, viewType,
- indexId, indexType, rowKeyOrderOptimizable);
+ indexId, indexType, rowKeyOrderOptimizable, transactional);
result = new MetaDataMutationResult(code, result.getMutationTime(), table, true);
addTableToCache(result);
return table;
@@ -2175,6 +2295,8 @@ public class MetaDataClient {
MetaDataMutationResult result = connection.getQueryServices().dropTable(tableMetaData, tableType, cascade);
MutationCode code = result.getMutationCode();
+ PTable table = result.getTable();
+ boolean transactional = table!=null && table.isTransactional();
switch (code) {
case TABLE_NOT_FOUND:
if (!ifExists) { throw new TableNotFoundException(schemaName, tableName); }
@@ -2186,12 +2308,10 @@ public class MetaDataClient {
.setSchemaName(schemaName).setTableName(tableName).build().buildException();
default:
- connection.removeTable(tenantId, SchemaUtil.getTableName(schemaName, tableName), parentTableName,
- result.getMutationTime());
+ connection.removeTable(tenantId, SchemaUtil.getTableName(schemaName, tableName), parentTableName, result.getMutationTime());
if (result.getTable() != null && tableType != PTableType.VIEW) {
connection.setAutoCommit(true);
- PTable table = result.getTable();
boolean dropMetaData = result.getTable().getViewIndexId() == null &&
connection.getQueryServices().getProps().getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA);
long ts = (scn == null ? result.getMutationTime() : scn);
@@ -2334,12 +2454,12 @@ public class MetaDataClient {
return mutationCode;
}
- private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta) throws SQLException {
- return incrementTableSeqNum(table, expectedType, columnCountDelta, null, null, null, null);
+ private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta, Boolean isTransactional) throws SQLException {
+ return incrementTableSeqNum(table, expectedType, columnCountDelta, isTransactional, null, null, null, null);
}
private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta,
- Boolean isImmutableRows, Boolean disableWAL, Boolean isMultiTenant, Boolean storeNulls)
+ Boolean isTransactional, Boolean isImmutableRows, Boolean disableWAL, Boolean isMultiTenant, Boolean storeNulls)
throws SQLException {
String schemaName = table.getSchemaName().getString();
String tableName = table.getTableName().getString();
@@ -2371,6 +2491,9 @@ public class MetaDataClient {
if (storeNulls != null) {
mutateBooleanProperty(tenantId, schemaName, tableName, STORE_NULLS, storeNulls);
}
+ if (isTransactional != null) {
+ mutateBooleanProperty(tenantId, schemaName, tableName, TRANSACTIONAL, isTransactional);
+ }
return seqNum;
}
@@ -2404,10 +2527,13 @@ public class MetaDataClient {
Boolean multiTenantProp = null;
Boolean disableWALProp = null;
Boolean storeNullsProp = null;
+ Boolean isTransactionalProp = null;
ListMultimap<String,Pair<String,Object>> stmtProperties = statement.getProps();
Map<String, List<Pair<String, Object>>> properties = new HashMap<>(stmtProperties.size());
- PTable table = FromCompiler.getResolver(statement, connection).getTables().get(0).getTable();
+ TableRef tableRef = FromCompiler.getResolver(statement, connection).getTables().get(0);
+ PTable table = tableRef.getTable();
+ Long timeStamp = table.isTransactional() ? tableRef.getTimeStamp() : null;
List<ColumnDef> columnDefs = statement.getColumnDefs();
if (columnDefs == null) {
columnDefs = Collections.emptyList();
@@ -2426,6 +2552,8 @@ public class MetaDataClient {
disableWALProp = (Boolean)prop.getSecond();
} else if (propName.equals(STORE_NULLS)) {
storeNullsProp = (Boolean)prop.getSecond();
+ } else if (propName.equals(TRANSACTIONAL)) {
+ isTransactionalProp = (Boolean)prop.getSecond();
}
}
}
@@ -2433,6 +2561,7 @@ public class MetaDataClient {
}
boolean retried = false;
boolean changingPhoenixTableProperty = false;
+ boolean nonTxToTx = false;
while (true) {
ColumnResolver resolver = FromCompiler.getResolver(statement, connection);
table = resolver.getTables().get(0).getTable();
@@ -2487,6 +2616,23 @@ public class MetaDataClient {
changingPhoenixTableProperty = true;
}
}
+ Boolean isTransactional = null;
+ if (isTransactionalProp != null) {
+ if (isTransactionalProp.booleanValue() != table.isTransactional()) {
+ isTransactional = isTransactionalProp;
+ // We can only go one way: from non transactional to transactional
+ // Going the other way would require rewriting the cell timestamps
+ // and doing a major compaction to get rid of any Tephra specific
+ // delete markers.
+ if (!isTransactional) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.TX_MAY_NOT_SWITCH_TO_NON_TX)
+ .setSchemaName(schemaName).setTableName(tableName).build().buildException();
+ }
+ timeStamp = TransactionUtil.getTableTimestamp(connection, isTransactional);
+ changingPhoenixTableProperty = true;
+ nonTxToTx = true;
+ }
+ }
int numPkColumnsAdded = 0;
PreparedStatement colUpsert = connection.prepareStatement(INSERT_COLUMN_ALTER_TABLE);
@@ -2558,7 +2704,7 @@ public class MetaDataClient {
}
}
- columnMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ columnMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
connection.rollback();
} else {
// Check that HBase configured properly for mutable secondary indexing
@@ -2567,10 +2713,12 @@ public class MetaDataClient {
if (Boolean.FALSE.equals(isImmutableRows) && !table.getIndexes().isEmpty()) {
int hbaseVersion = connection.getQueryServices().getLowestClusterHBaseVersion();
if (hbaseVersion < PhoenixDatabaseMetaData.MUTABLE_SI_VERSION_THRESHOLD) {
- throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_MUTABLE_INDEXES).setSchemaName(schemaName).setTableName(tableName).build().buildException();
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_MUTABLE_INDEXES)
+ .setSchemaName(schemaName).setTableName(tableName).build().buildException();
}
- if (connection.getQueryServices().hasInvalidIndexConfiguration()) {
- throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_MUTABLE_INDEX_CONFIG).setSchemaName(schemaName).setTableName(tableName).build().buildException();
+ if (connection.getQueryServices().isMutableIndexWALCodecInstalled() && !table.isTransactional()) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_MUTABLE_INDEX_CONFIG)
+ .setSchemaName(schemaName).setTableName(tableName).build().buildException();
}
}
if (Boolean.TRUE.equals(multiTenant)) {
@@ -2578,17 +2726,17 @@ public class MetaDataClient {
}
}
- if (numPkColumnsAdded>0 && !table.getIndexes().isEmpty()) {
+ if (!table.getIndexes().isEmpty() && (numPkColumnsAdded>0 || nonTxToTx)) {
for (PTable index : table.getIndexes()) {
- incrementTableSeqNum(index, index.getType(), numPkColumnsAdded);
+ incrementTableSeqNum(index, index.getType(), numPkColumnsAdded, nonTxToTx ? Boolean.TRUE : null);
}
- tableMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
connection.rollback();
}
long seqNum = table.getSequenceNumber();
if (changingPhoenixTableProperty || columnDefs.size() > 0) {
- seqNum = incrementTableSeqNum(table, statement.getTableType(), columnDefs.size(), isImmutableRows, disableWAL, multiTenant, storeNulls);
- tableMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ seqNum = incrementTableSeqNum(table, statement.getTableType(), columnDefs.size(), isTransactional, isImmutableRows, disableWAL, multiTenant, storeNulls);
+ tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
connection.rollback();
}
@@ -2626,10 +2774,22 @@ public class MetaDataClient {
return new MutationState(0,connection);
}
- // Only update client side cache if we aren't adding a PK column to a table with indexes.
+ // Only update client side cache if we aren't adding a PK column to a table with indexes or
+ // transitioning a table from non transactional to transactional.
// We could update the cache manually then too, it'd just be a pain.
- if (numPkColumnsAdded==0 || table.getIndexes().isEmpty()) {
- connection.addColumn(tenantId, SchemaUtil.getTableName(schemaName, tableName), columns, result.getMutationTime(), seqNum, isImmutableRows == null ? table.isImmutableRows() : isImmutableRows, disableWAL == null ? table.isWALDisabled() : disableWAL, multiTenant == null ? table.isMultiTenant() : multiTenant, storeNulls == null ? table.getStoreNulls() : storeNulls);
+ if (table.getIndexes().isEmpty() || (numPkColumnsAdded==0 && !nonTxToTx)) {
+ connection.addColumn(
+ tenantId,
+ SchemaUtil.getTableName(schemaName, tableName),
+ columns,
+ result.getMutationTime(),
+ seqNum,
+ isImmutableRows == null ? table.isImmutableRows() : isImmutableRows,
+ disableWAL == null ? table.isWALDisabled() : disableWAL,
+ multiTenant == null ? table.isMultiTenant() : multiTenant,
+ storeNulls == null ? table.getStoreNulls() : storeNulls,
+ isTransactional == null ? table.isTransactional() : isTransactional,
+ TransactionUtil.getResolvedTime(connection, result));
}
// Delete rows in view index if we haven't dropped it already
// We only need to do this if the multiTenant transitioned to false
@@ -2760,12 +2920,12 @@ public class MetaDataClient {
boolean retried = false;
while (true) {
final ColumnResolver resolver = FromCompiler.getResolver(statement, connection);
- PTable table = resolver.getTables().get(0).getTable();
+ TableRef tableRef = resolver.getTables().get(0);
+ PTable table = tableRef.getTable();
List<ColumnName> columnRefs = statement.getColumnRefs();
if(columnRefs == null) {
columnRefs = Lists.newArrayListWithCapacity(0);
}
- TableRef tableRef = null;
List<ColumnRef> columnsToDrop = Lists.newArrayListWithExpectedSize(columnRefs.size() + table.getIndexes().size());
List<TableRef> indexesToDrop = Lists.newArrayListWithExpectedSize(table.getIndexes().size());
List<Mutation> tableMetaData = Lists.newArrayListWithExpectedSize((table.getIndexes().size() + 1) * (1 + table.getColumns().size() - columnRefs.size()));
@@ -2781,14 +2941,13 @@ public class MetaDataClient {
}
throw e;
}
- tableRef = columnRef.getTableRef();
PColumn columnToDrop = columnRef.getColumn();
tableColumnsToDrop.add(columnToDrop);
if (SchemaUtil.isPKColumn(columnToDrop)) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_DROP_PK)
.setColumnName(columnToDrop.getName().getString()).build().buildException();
}
- columnsToDrop.add(new ColumnRef(tableRef, columnToDrop.getPosition()));
+ columnsToDrop.add(new ColumnRef(columnRef.getTableRef(), columnToDrop.getPosition()));
}
dropColumnMutations(table, tableColumnsToDrop, tableMetaData);
@@ -2810,16 +2969,17 @@ public class MetaDataClient {
}
}
if(!indexColumnsToDrop.isEmpty()) {
- incrementTableSeqNum(index, index.getType(), -indexColumnsToDrop.size());
+ incrementTableSeqNum(index, index.getType(), -indexColumnsToDrop.size(), null);
dropColumnMutations(index, indexColumnsToDrop, tableMetaData);
}
}
- tableMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ Long timeStamp = table.isTransactional() ? tableRef.getTimeStamp() : null;
+ tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
connection.rollback();
- long seqNum = incrementTableSeqNum(table, statement.getTableType(), -tableColumnsToDrop.size());
- tableMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ long seqNum = incrementTableSeqNum(table, statement.getTableType(), -tableColumnsToDrop.size(), null);
+ tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
connection.rollback();
// Force table header to be first in list
Collections.reverse(tableMetaData);
@@ -2870,7 +3030,7 @@ public class MetaDataClient {
// client-side cache as it would be too painful. Just let it pull it over from
// the server when needed.
if (tableColumnsToDrop.size() > 0 && indexesToDrop.isEmpty()) {
- connection.removeColumn(tenantId, SchemaUtil.getTableName(schemaName, tableName) , tableColumnsToDrop, result.getMutationTime(), seqNum);
+ connection.removeColumn(tenantId, SchemaUtil.getTableName(schemaName, tableName) , tableColumnsToDrop, result.getMutationTime(), seqNum, TransactionUtil.getResolvedTime(connection, result));
}
// If we have a VIEW, then only delete the metadata, and leave the table data alone
if (table.getType() != PTableType.VIEW) {
@@ -2955,7 +3115,8 @@ public class MetaDataClient {
tableUpsert.close();
}
}
- List<Mutation> tableMetadata = connection.getMutationState().toMutations().next().getSecond();
+ Long timeStamp = indexRef.getTable().isTransactional() ? indexRef.getTimeStamp() : null;
+ List<Mutation> tableMetadata = connection.getMutationState().toMutations(timeStamp).next().getSecond();
connection.rollback();
MetaDataMutationResult result = connection.getQueryServices().updateIndexState(tableMetadata, dataTableName);
@@ -3003,9 +3164,9 @@ public class MetaDataClient {
}
private PTable addTableToCache(MetaDataMutationResult result) throws SQLException {
- addIndexesFromPhysicalTable(result);
+ addIndexesFromPhysicalTable(result, null);
PTable table = result.getTable();
- connection.addTable(table);
+ connection.addTable(table, TransactionUtil.getResolvedTime(connection, result));
return table;
}
@@ -3025,7 +3186,8 @@ public class MetaDataClient {
*/
boolean isSharedIndex = table.getViewIndexId() != null;
if (isSharedIndex) {
- return connection.getQueryServices().getTableStats(table.getPhysicalName().getBytes(), getClientTimeStamp());
+ // we are assuming the stats table is not transactional
+ return connection.getQueryServices().getTableStats(table.getPhysicalName().getBytes(), getCurrentScn());
}
boolean isView = table.getType() == PTableType.VIEW;
String physicalName = table.getPhysicalName().getString();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java
index f015177..a3103bf 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java
@@ -17,6 +17,8 @@
*/
package org.apache.phoenix.schema;
+import java.sql.SQLException;
+
import org.apache.phoenix.parse.PFunction;
import org.apache.phoenix.query.MetaDataMutated;
@@ -28,7 +30,7 @@ public interface PMetaData extends MetaDataMutated, Iterable<PTable>, Cloneable
}
public int size();
public PMetaData clone();
- public PTable getTable(PTableKey key) throws TableNotFoundException;
+ public PTableRef getTableRef(PTableKey key) throws TableNotFoundException;
public PMetaData pruneTables(Pruner pruner);
public PFunction getFunction(PTableKey key) throws FunctionNotFoundException;
public PMetaData pruneFunctions(Pruner pruner);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
index a15de13..9e4460d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
@@ -41,32 +41,12 @@ import com.google.common.primitives.Longs;
*
*/
public class PMetaDataImpl implements PMetaData {
- private static final class PTableRef {
- public final PTable table;
- public final int estSize;
- public volatile long lastAccessTime;
-
- public PTableRef(PTable table, long lastAccessTime, int estSize) {
- this.table = table;
- this.lastAccessTime = lastAccessTime;
- this.estSize = estSize;
- }
-
- public PTableRef(PTable table, long lastAccessTime) {
- this (table, lastAccessTime, table.getEstimatedSize());
- }
-
- public PTableRef(PTableRef tableRef) {
- this (tableRef.table, tableRef.lastAccessTime, tableRef.estSize);
- }
- }
-
private static class PMetaDataCache implements Cloneable {
private static final int MIN_REMOVAL_SIZE = 3;
private static final Comparator<PTableRef> COMPARATOR = new Comparator<PTableRef>() {
@Override
public int compare(PTableRef tableRef1, PTableRef tableRef2) {
- return Longs.compare(tableRef1.lastAccessTime, tableRef2.lastAccessTime);
+ return Longs.compare(tableRef1.getLastAccessTime(), tableRef2.getLastAccessTime());
}
};
private static final MinMaxPriorityQueue.Builder<PTableRef> BUILDER = MinMaxPriorityQueue.orderedBy(COMPARATOR);
@@ -97,7 +77,7 @@ public class PMetaDataImpl implements PMetaData {
Map<PTableKey,PTableRef> newTables = newMap(Math.max(tables.size(),expectedCapacity));
// Copy value so that access time isn't changing anymore
for (PTableRef tableAccess : tables.values()) {
- newTables.put(tableAccess.table.getKey(), new PTableRef(tableAccess));
+ newTables.put(tableAccess.getTable().getKey(), new PTableRef(tableAccess));
}
return newTables;
}
@@ -133,7 +113,7 @@ public class PMetaDataImpl implements PMetaData {
if (tableAccess == null) {
return null;
}
- tableAccess.lastAccessTime = timeKeeper.getCurrentTime();
+ tableAccess.setLastAccessTime(timeKeeper.getCurrentTime());
return tableAccess;
}
@@ -157,37 +137,37 @@ public class PMetaDataImpl implements PMetaData {
// Add to new cache, but track references to remove when done
// to bring cache at least overage amount below it's max size.
for (PTableRef tableRef : this.tables.values()) {
- newCache.put(tableRef.table.getKey(), new PTableRef(tableRef));
+ newCache.put(tableRef.getTable().getKey(), new PTableRef(tableRef));
toRemove.add(tableRef);
- toRemoveBytes += tableRef.estSize;
- while (toRemoveBytes - toRemove.peekLast().estSize >= overage) {
+ toRemoveBytes += tableRef.getEstSize();
+ while (toRemoveBytes - toRemove.peekLast().getEstSize() >= overage) {
PTableRef removedRef = toRemove.removeLast();
- toRemoveBytes -= removedRef.estSize;
+ toRemoveBytes -= removedRef.getEstSize();
}
}
for (PTableRef toRemoveRef : toRemove) {
- newCache.remove(toRemoveRef.table.getKey());
+ newCache.remove(toRemoveRef.getTable().getKey());
}
return newCache;
}
private PTable put(PTableKey key, PTableRef ref) {
- currentByteSize += ref.estSize;
+ currentByteSize += ref.getEstSize();
PTableRef oldTableAccess = this.tables.put(key, ref);
PTable oldTable = null;
if (oldTableAccess != null) {
- currentByteSize -= oldTableAccess.estSize;
- oldTable = oldTableAccess.table;
+ currentByteSize -= oldTableAccess.getEstSize();
+ oldTable = oldTableAccess.getTable();
}
return oldTable;
}
- public PTable put(PTableKey key, PTable value) {
- return put(key, new PTableRef(value, timeKeeper.getCurrentTime()));
+ public PTable put(PTableKey key, PTable value, long resolvedTime) {
+ return put(key, new PTableRef(value, timeKeeper.getCurrentTime(), resolvedTime));
}
- public PTable putDuplicate(PTableKey key, PTable value) {
- return put(key, new PTableRef(value, timeKeeper.getCurrentTime(), 0));
+ public PTable putDuplicate(PTableKey key, PTable value, long resolvedTime) {
+ return put(key, new PTableRef(value, timeKeeper.getCurrentTime(), 0, resolvedTime));
}
public PTable remove(PTableKey key) {
@@ -195,8 +175,8 @@ public class PMetaDataImpl implements PMetaData {
if (value == null) {
return null;
}
- currentByteSize -= value.estSize;
- return value.table;
+ currentByteSize -= value.getEstSize();
+ return value.getTable();
}
public Iterator<PTable> iterator() {
@@ -210,7 +190,7 @@ public class PMetaDataImpl implements PMetaData {
@Override
public PTable next() {
- return iterator.next().table;
+ return iterator.next().getTable();
}
@Override
@@ -254,12 +234,12 @@ public class PMetaDataImpl implements PMetaData {
}
@Override
- public PTable getTable(PTableKey key) throws TableNotFoundException {
+ public PTableRef getTableRef(PTableKey key) throws TableNotFoundException {
PTableRef ref = metaData.get(key);
if (ref == null) {
throw new TableNotFoundException(key.getName());
}
- return ref.table;
+ return ref;
}
@Override
@@ -276,22 +256,29 @@ public class PMetaDataImpl implements PMetaData {
return metaData.size();
}
+ @Override
+ public PMetaData updateResolvedTimestamp(PTable table, long resolvedTimestamp) throws SQLException {
+ PMetaDataCache clone = metaData.clone();
+ clone.putDuplicate(table.getKey(), table, resolvedTimestamp);
+ return new PMetaDataImpl(clone);
+ }
@Override
- public PMetaData addTable(PTable table) throws SQLException {
+ public PMetaData addTable(PTable table, long resolvedTime) throws SQLException {
int netGain = 0;
PTableKey key = table.getKey();
PTableRef oldTableRef = metaData.get(key);
if (oldTableRef != null) {
- netGain -= oldTableRef.estSize;
+ netGain -= oldTableRef.getEstSize();
}
PTable newParentTable = null;
+ long parentResolvedTimestamp = resolvedTime;
if (table.getParentName() != null) { // Upsert new index table into parent data table list
String parentName = table.getParentName().getString();
PTableRef oldParentRef = metaData.get(new PTableKey(table.getTenantId(), parentName));
// If parentTable isn't cached, that's ok we can skip this
if (oldParentRef != null) {
- List<PTable> oldIndexes = oldParentRef.table.getIndexes();
+ List<PTable> oldIndexes = oldParentRef.getTable().getIndexes();
List<PTable> newIndexes = Lists.newArrayListWithExpectedSize(oldIndexes.size() + 1);
newIndexes.addAll(oldIndexes);
for (int i = 0; i < newIndexes.size(); i++) {
@@ -302,8 +289,8 @@ public class PMetaDataImpl implements PMetaData {
}
}
newIndexes.add(table);
- netGain -= oldParentRef.estSize;
- newParentTable = PTableImpl.makePTable(oldParentRef.table, table.getTimeStamp(), newIndexes);
+ netGain -= oldParentRef.getEstSize();
+ newParentTable = PTableImpl.makePTable(oldParentRef.getTable(), table.getTimeStamp(), newIndexes);
netGain += newParentTable.getEstimatedSize();
}
}
@@ -314,24 +301,24 @@ public class PMetaDataImpl implements PMetaData {
PMetaDataCache newMetaData = overage <= 0 ? metaData.clone() : metaData.cloneMinusOverage(overage);
if (newParentTable != null) { // Upsert new index table into parent data table list
- newMetaData.put(newParentTable.getKey(), newParentTable);
- newMetaData.putDuplicate(table.getKey(), table);
+ newMetaData.put(newParentTable.getKey(), newParentTable, parentResolvedTimestamp);
+ newMetaData.putDuplicate(table.getKey(), table, resolvedTime);
} else {
- newMetaData.put(table.getKey(), table);
+ newMetaData.put(table.getKey(), table, resolvedTime);
}
for (PTable index : table.getIndexes()) {
- newMetaData.putDuplicate(index.getKey(), index);
+ newMetaData.putDuplicate(index.getKey(), index, resolvedTime);
}
return new PMetaDataImpl(newMetaData);
}
@Override
- public PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columnsToAdd, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls) throws SQLException {
+ public PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columnsToAdd, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException {
PTableRef oldTableRef = metaData.get(new PTableKey(tenantId, tableName));
if (oldTableRef == null) {
return this;
}
- List<PColumn> oldColumns = PTableImpl.getColumnsToClone(oldTableRef.table);
+ List<PColumn> oldColumns = PTableImpl.getColumnsToClone(oldTableRef.getTable());
List<PColumn> newColumns;
if (columnsToAdd.isEmpty()) {
newColumns = oldColumns;
@@ -340,8 +327,8 @@ public class PMetaDataImpl implements PMetaData {
newColumns.addAll(oldColumns);
newColumns.addAll(columnsToAdd);
}
- PTable newTable = PTableImpl.makePTable(oldTableRef.table, tableTimeStamp, tableSeqNum, newColumns, isImmutableRows, isWalDisabled, isMultitenant, storeNulls);
- return addTable(newTable);
+ PTable newTable = PTableImpl.makePTable(oldTableRef.getTable(), tableTimeStamp, tableSeqNum, newColumns, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional);
+ return addTable(newTable, resolvedTime);
}
@Override
@@ -368,7 +355,7 @@ public class PMetaDataImpl implements PMetaData {
}
// also remove its reference from parent table
if (parentTableRef != null) {
- List<PTable> oldIndexes = parentTableRef.table.getIndexes();
+ List<PTable> oldIndexes = parentTableRef.getTable().getIndexes();
if(oldIndexes != null && !oldIndexes.isEmpty()) {
List<PTable> newIndexes = Lists.newArrayListWithExpectedSize(oldIndexes.size());
newIndexes.addAll(oldIndexes);
@@ -377,13 +364,13 @@ public class PMetaDataImpl implements PMetaData {
if (index.getName().getString().equals(tableName)) {
newIndexes.remove(i);
PTable parentTable = PTableImpl.makePTable(
- parentTableRef.table,
- tableTimeStamp == HConstants.LATEST_TIMESTAMP ? parentTableRef.table.getTimeStamp() : tableTimeStamp,
+ parentTableRef.getTable(),
+ tableTimeStamp == HConstants.LATEST_TIMESTAMP ? parentTableRef.getTable().getTimeStamp() : tableTimeStamp,
newIndexes);
if (tables == null) {
tables = metaData.clone();
}
- tables.put(parentTable.getKey(), parentTable);
+ tables.put(parentTable.getKey(), parentTable, parentTableRef.getResolvedTimeStamp());
break;
}
}
@@ -393,12 +380,12 @@ public class PMetaDataImpl implements PMetaData {
}
@Override
- public PMetaData removeColumn(PName tenantId, String tableName, List<PColumn> columnsToRemove, long tableTimeStamp, long tableSeqNum) throws SQLException {
+ public PMetaData removeColumn(PName tenantId, String tableName, List<PColumn> columnsToRemove, long tableTimeStamp, long tableSeqNum, long resolvedTime) throws SQLException {
PTableRef tableRef = metaData.get(new PTableKey(tenantId, tableName));
if (tableRef == null) {
return this;
}
- PTable table = tableRef.table;
+ PTable table = tableRef.getTable();
PMetaDataCache tables = metaData.clone();
for (PColumn columnToRemove : columnsToRemove) {
PColumn column;
@@ -427,7 +414,7 @@ public class PMetaDataImpl implements PMetaData {
table = PTableImpl.makePTable(table, tableTimeStamp, tableSeqNum, columns);
}
- tables.put(table.getKey(), table);
+ tables.put(table.getKey(), table, resolvedTime);
return new PMetaDataImpl(tables);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
index a2979d4..ec97394 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
@@ -307,13 +307,14 @@ public interface PTable extends PMetaDataEntity {
PName getPhysicalName();
boolean isImmutableRows();
- void getIndexMaintainers(ImmutableBytesWritable ptr, PhoenixConnection connection);
+ boolean getIndexMaintainers(ImmutableBytesWritable ptr, PhoenixConnection connection);
IndexMaintainer getIndexMaintainer(PTable dataTable, PhoenixConnection connection);
PName getDefaultFamilyName();
boolean isWALDisabled();
boolean isMultiTenant();
boolean getStoreNulls();
+ boolean isTransactional();
ViewType getViewType();
String getViewStatement();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index ff4b512..0827ea7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -77,6 +77,8 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.sun.istack.NotNull;
+import co.cask.tephra.TxConstants;
+
/**
*
* Base class for PTable implementors. Provides abstraction for
@@ -123,6 +125,7 @@ public class PTableImpl implements PTable {
private boolean disableWAL;
private boolean multiTenant;
private boolean storeNulls;
+ private boolean isTransactional;
private ViewType viewType;
private Short viewIndexId;
private int estimatedSize;
@@ -202,7 +205,7 @@ public class PTableImpl implements PTable {
table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), getColumnsToClone(table), parentSchemaName, table.getParentTableName(),
indexes, table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), viewStatement,
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
- table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable());
+ table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional());
}
public static PTableImpl makePTable(PTable table, List<PColumn> columns) throws SQLException {
@@ -211,7 +214,7 @@ public class PTableImpl implements PTable {
table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(),
table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
- table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable());
+ table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional());
}
public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List<PColumn> columns) throws SQLException {
@@ -220,7 +223,7 @@ public class PTableImpl implements PTable {
sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(),
table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(),
table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(),
- table.getBaseColumnCount(), table.rowKeyOrderOptimizable());
+ table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional());
}
public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List<PColumn> columns, boolean isImmutableRows) throws SQLException {
@@ -229,16 +232,16 @@ public class PTableImpl implements PTable {
sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(),
table.getIndexes(), isImmutableRows, table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(),
- table.getIndexType(), table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable());
+ table.getIndexType(), table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional());
}
- public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List<PColumn> columns, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls) throws SQLException {
+ public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List<PColumn> columns, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional) throws SQLException {
return new PTableImpl(
table.getTenantId(), table.getSchemaName(), table.getTableName(), table.getType(), table.getIndexState(), timeStamp,
sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(),
table.getIndexes(), isImmutableRows, table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
isWalDisabled, isMultitenant, storeNulls, table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(),
- table.getBaseColumnCount(), table.rowKeyOrderOptimizable());
+ table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), isTransactional);
}
public static PTableImpl makePTable(PTable table, PIndexState state) throws SQLException {
@@ -248,7 +251,7 @@ public class PTableImpl implements PTable {
table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(),
table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
- table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable());
+ table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional());
}
public static PTableImpl makePTable(PTable table, boolean rowKeyOrderOptimizable) throws SQLException {
@@ -258,7 +261,7 @@ public class PTableImpl implements PTable {
table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(),
table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(),
- table.getBaseColumnCount(), rowKeyOrderOptimizable);
+ table.getBaseColumnCount(), rowKeyOrderOptimizable, table.isTransactional());
}
public static PTableImpl makePTable(PTable table, PTableStats stats) throws SQLException {
@@ -268,28 +271,28 @@ public class PTableImpl implements PTable {
table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(),
table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), stats,
- table.getBaseColumnCount(), table.rowKeyOrderOptimizable());
+ table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional());
}
public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, long timeStamp, long sequenceNumber,
PName pkName, Integer bucketNum, List<PColumn> columns, PName dataSchemaName, PName dataTableName, List<PTable> indexes,
boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant,
- boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, boolean rowKeyOrderOptimizable) throws SQLException {
+ boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, boolean rowKeyOrderOptimizable, boolean isTransactional) throws SQLException {
return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, dataSchemaName,
dataTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName,
viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId,
- indexType, PTableStats.EMPTY_STATS, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, rowKeyOrderOptimizable);
+ indexType, PTableStats.EMPTY_STATS, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, rowKeyOrderOptimizable, isTransactional);
}
public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type,
PIndexState state, long timeStamp, long sequenceNumber, PName pkName, Integer bucketNum,
List<PColumn> columns, PName dataSchemaName, PName dataTableName, List<PTable> indexes,
boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression,
- boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, @NotNull PTableStats stats, int baseColumnCount, boolean rowKeyOrderOptimizable)
+ boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, @NotNull PTableStats stats, int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional)
throws SQLException {
return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName,
bucketNum, columns, dataSchemaName, dataTableName, indexes, isImmutableRows, physicalNames,
- defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, stats, baseColumnCount, rowKeyOrderOptimizable);
+ defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, stats, baseColumnCount, rowKeyOrderOptimizable, isTransactional);
}
private PTableImpl(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state,
@@ -297,10 +300,10 @@ public class PTableImpl implements PTable {
PName parentSchemaName, PName parentTableName, List<PTable> indexes, boolean isImmutableRows,
List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant,
boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType,
- PTableStats stats, int baseColumnCount, boolean rowKeyOrderOptimizable) throws SQLException {
+ PTableStats stats, int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional) throws SQLException {
init(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns,
stats, schemaName, parentTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName,
- viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable);
+ viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, isTransactional);
}
@Override
@@ -328,7 +331,7 @@ public class PTableImpl implements PTable {
PName pkName, Integer bucketNum, List<PColumn> columns, PTableStats stats, PName parentSchemaName, PName parentTableName,
List<PTable> indexes, boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL,
boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId,
- IndexType indexType , int baseColumnCount, boolean rowKeyOrderOptimizable) throws SQLException {
+ IndexType indexType , int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional) throws SQLException {
Preconditions.checkNotNull(schemaName);
Preconditions.checkArgument(tenantId==null || tenantId.getBytes().length > 0); // tenantId should be null or not empty
int estimatedSize = SizedUtil.OBJECT_SIZE * 2 + 23 * SizedUtil.POINTER_SIZE + 4 * SizedUtil.INT_SIZE + 2 * SizedUtil.LONG_SIZE + 2 * SizedUtil.INT_OBJECT_SIZE +
@@ -357,6 +360,7 @@ public class PTableImpl implements PTable {
this.viewType = viewType;
this.viewIndexId = viewIndexId;
this.indexType = indexType;
+ this.isTransactional = isTransactional;
this.tableStats = stats;
this.rowKeyOrderOptimizable = rowKeyOrderOptimizable;
List<PColumn> pkColumns;
@@ -676,7 +680,7 @@ public class PTableImpl implements PTable {
private Put setValues;
private Delete unsetValues;
- private Delete deleteRow;
+ private Mutation deleteRow;
private final long ts;
public PRowImpl(KeyValueBuilder kvBuilder, ImmutableBytesWritable key, long ts, Integer bucketNum) {
@@ -713,7 +717,8 @@ public class PTableImpl implements PTable {
// way HBase works.
addQuietly(setValues, kvBuilder, kvBuilder.buildPut(keyPtr,
SchemaUtil.getEmptyColumnFamilyPtr(PTableImpl.this),
- QueryConstants.EMPTY_COLUMN_BYTES_PTR, ts, ByteUtil.EMPTY_BYTE_ARRAY_PTR));
+ QueryConstants.EMPTY_COLUMN_BYTES_PTR, ts,
+ QueryConstants.EMPTY_COLUMN_VALUE_BYTES_PTR));
mutations.add(setValues);
if (!unsetValues.isEmpty()) {
mutations.add(unsetValues);
@@ -779,11 +784,22 @@ public class PTableImpl implements PTable {
@Override
public void delete() {
newMutations();
- Delete delete = new Delete(key);
- for (PColumnFamily colFamily : families) {
- delete.addFamily(colFamily.getName().getBytes(), ts);
+ // we're using the Tephra column family delete marker here to prevent the translation
+ // of deletes to puts by the Tephra's TransactionProcessor
+ if (PTableImpl.this.isTransactional()) {
+ Put delete = new Put(key);
+ for (PColumnFamily colFamily : families) {
+ delete.add(colFamily.getName().getBytes(), TxConstants.FAMILY_DELETE_QUALIFIER, ts,
+ HConstants.EMPTY_BYTE_ARRAY);
+ }
+ deleteRow = delete;
+ } else {
+ Delete delete = new Delete(key);
+ for (PColumnFamily colFamily : families) {
+ delete.deleteFamily(colFamily.getName().getBytes(), ts);
+ }
+ deleteRow = delete;
}
- deleteRow = delete;
// No need to write to the WAL for indexes
if (PTableImpl.this.getType() == PTableType.INDEX) {
deleteRow.setDurability(Durability.SKIP_WAL);
@@ -888,8 +904,8 @@ public class PTableImpl implements PTable {
}
@Override
- public synchronized void getIndexMaintainers(ImmutableBytesWritable ptr, PhoenixConnection connection) {
- if (indexMaintainersPtr == null) {
+ public synchronized boolean getIndexMaintainers(ImmutableBytesWritable ptr, PhoenixConnection connection) {
+ if (indexMaintainersPtr == null || indexMaintainersPtr.getLength()==0) {
indexMaintainersPtr = new ImmutableBytesWritable();
if (indexes.isEmpty()) {
indexMaintainersPtr.set(ByteUtil.EMPTY_BYTE_ARRAY);
@@ -898,6 +914,7 @@ public class PTableImpl implements PTable {
}
}
ptr.set(indexMaintainersPtr.get(), indexMaintainersPtr.getOffset(), indexMaintainersPtr.getLength());
+ return indexMaintainersPtr.getLength() > 0;
}
@Override
@@ -1014,6 +1031,7 @@ public class PTableImpl implements PTable {
boolean disableWAL = table.getDisableWAL();
boolean multiTenant = table.getMultiTenant();
boolean storeNulls = table.getStoreNulls();
+ boolean isTransactional = table.getTransactional();
ViewType viewType = null;
String viewStatement = null;
List<PName> physicalNames = Collections.emptyList();
@@ -1044,7 +1062,7 @@ public class PTableImpl implements PTable {
result.init(tenantId, schemaName, tableName, tableType, indexState, timeStamp, sequenceNumber, pkName,
(bucketNum == NO_SALTING) ? null : bucketNum, columns, stats, schemaName,dataTableName, indexes,
isImmutableRows, physicalNames, defaultFamilyName, viewStatement, disableWAL,
- multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable);
+ multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, isTransactional);
return result;
} catch (SQLException e) {
throw new RuntimeException(e); // Impossible
@@ -1124,6 +1142,7 @@ public class PTableImpl implements PTable {
builder.setDisableWAL(table.isWALDisabled());
builder.setMultiTenant(table.isMultiTenant());
builder.setStoreNulls(table.getStoreNulls());
+ builder.setTransactional(table.isTransactional());
if(table.getType() == PTableType.VIEW){
builder.setViewType(ByteStringer.wrap(new byte[]{table.getViewType().getSerializedValue()}));
builder.setViewStatement(ByteStringer.wrap(PVarchar.INSTANCE.toBytes(table.getViewStatement())));
@@ -1155,6 +1174,10 @@ public class PTableImpl implements PTable {
}
@Override
+ public boolean isTransactional() {
+ return isTransactional;
+ }
+
public int getBaseColumnCount() {
return baseColumnCount;
}
[4/9] phoenix git commit: PHOENIX-1674 Snapshot isolation transaction
support through Tephra (James Taylor, Thomas D'Silva)
Posted by td...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
new file mode 100644
index 0000000..909d59b
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.index;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellScanner;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
+import org.apache.phoenix.compile.ScanRanges;
+import org.apache.phoenix.hbase.index.MultiMutation;
+import org.apache.phoenix.hbase.index.ValueGetter;
+import org.apache.phoenix.hbase.index.covered.IndexUpdate;
+import org.apache.phoenix.hbase.index.covered.TableState;
+import org.apache.phoenix.hbase.index.covered.update.ColumnReference;
+import org.apache.phoenix.hbase.index.covered.update.ColumnTracker;
+import org.apache.phoenix.hbase.index.covered.update.IndexedColumnGroup;
+import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.hbase.index.write.IndexWriter;
+import org.apache.phoenix.query.KeyRange;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.types.PVarbinary;
+import org.apache.phoenix.trace.TracingUtils;
+import org.apache.phoenix.trace.util.NullSpan;
+import org.apache.phoenix.util.ScanUtil;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.ServerUtil;
+
+import co.cask.tephra.Transaction;
+import co.cask.tephra.Transaction.VisibilityLevel;
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.hbase11.TransactionAwareHTable;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.primitives.Longs;
+
+/**
+ * Do all the work of managing index updates for a transactional table from a single coprocessor. Since the transaction
+ * manager essentially time orders writes through conflict detection, the logic to maintain a secondary index is quite a
+ * bit simpler than the non transactional case. For example, there's no need to muck with the WAL, as failure scenarios
+ * are handled by aborting the transaction.
+ */
+public class PhoenixTransactionalIndexer extends BaseRegionObserver {
+
+ private static final Log LOG = LogFactory.getLog(PhoenixTransactionalIndexer.class);
+
+ private PhoenixIndexCodec codec;
+ private IndexWriter writer;
+ private boolean stopped;
+
+ @Override
+ public void start(CoprocessorEnvironment e) throws IOException {
+ final RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment)e;
+ String serverName = env.getRegionServerServices().getServerName().getServerName();
+ codec = new PhoenixIndexCodec();
+ codec.initialize(env);
+
+ // setup the actual index writer
+ this.writer = new IndexWriter(env, serverName + "-tx-index-writer");
+ }
+
+ @Override
+ public void stop(CoprocessorEnvironment e) throws IOException {
+ if (this.stopped) { return; }
+ this.stopped = true;
+ String msg = "TxIndexer is being stopped";
+ this.writer.stop(msg);
+ }
+
+ private static Iterator<Mutation> getMutationIterator(final MiniBatchOperationInProgress<Mutation> miniBatchOp) {
+ return new Iterator<Mutation>() {
+ private int i = 0;
+
+ @Override
+ public boolean hasNext() {
+ return i < miniBatchOp.size();
+ }
+
+ @Override
+ public Mutation next() {
+ return miniBatchOp.getOperation(i++);
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+ @Override
+ public void preBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c,
+ MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
+
+ Mutation m = miniBatchOp.getOperation(0);
+ if (!codec.isEnabled(m)) {
+ super.preBatchMutate(c, miniBatchOp);
+ return;
+ }
+
+ Map<String,byte[]> updateAttributes = m.getAttributesMap();
+ PhoenixIndexMetaData indexMetaData = new PhoenixIndexMetaData(c.getEnvironment(),updateAttributes);
+ byte[] txRollbackAttribute = m.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY);
+ Collection<Pair<Mutation, byte[]>> indexUpdates = null;
+ // get the current span, or just use a null-span to avoid a bunch of if statements
+ try (TraceScope scope = Trace.startSpan("Starting to build index updates")) {
+ Span current = scope.getSpan();
+ if (current == null) {
+ current = NullSpan.INSTANCE;
+ }
+
+ // get the index updates for all elements in this batch
+ indexUpdates = getIndexUpdates(c.getEnvironment(), indexMetaData, getMutationIterator(miniBatchOp), txRollbackAttribute);
+
+ current.addTimelineAnnotation("Built index updates, doing preStep");
+ TracingUtils.addAnnotation(current, "index update count", indexUpdates.size());
+
+ // no index updates, so we are done
+ if (!indexUpdates.isEmpty()) {
+ this.writer.write(indexUpdates);
+ }
+ } catch (Throwable t) {
+ String msg = "Failed to update index with entries:" + indexUpdates;
+ LOG.error(msg, t);
+ ServerUtil.throwIOException(msg, t);
+ }
+ }
+
+ private Collection<Pair<Mutation, byte[]>> getIndexUpdates(RegionCoprocessorEnvironment env, PhoenixIndexMetaData indexMetaData, Iterator<Mutation> mutationIterator, byte[] txRollbackAttribute) throws IOException {
+ ResultScanner currentScanner = null;
+ TransactionAwareHTable txTable = null;
+ // Collect up all mutations in batch
+ Map<ImmutableBytesPtr, MultiMutation> mutations =
+ new HashMap<ImmutableBytesPtr, MultiMutation>();
+ while(mutationIterator.hasNext()) {
+ Mutation m = mutationIterator.next();
+ // add the mutation to the batch set
+ ImmutableBytesPtr row = new ImmutableBytesPtr(m.getRow());
+ MultiMutation stored = mutations.get(row);
+ // we haven't seen this row before, so add it
+ if (stored == null) {
+ stored = new MultiMutation(row);
+ mutations.put(row, stored);
+ }
+ stored.addAll(m);
+ }
+
+ // Collect the set of mutable ColumnReferences so that we can first
+ // run a scan to get the current state. We'll need this to delete
+ // the existing index rows.
+ Transaction tx = indexMetaData.getTransaction();
+ assert(tx != null);
+ List<IndexMaintainer> indexMaintainers = indexMetaData.getIndexMaintainers();
+ Set<ColumnReference> mutableColumns = Sets.newHashSetWithExpectedSize(indexMaintainers.size() * 10);
+ for (IndexMaintainer indexMaintainer : indexMaintainers) {
+ // For transactional tables, we use an index maintainer
+ // to aid in rollback if there's a KeyValue column in the index. The alternative would be
+ // to hold on to all uncommitted index row keys (even ones already sent to HBase) on the
+ // client side.
+ mutableColumns.addAll(indexMaintainer.getAllColumns());
+ }
+
+ boolean isRollback = txRollbackAttribute!=null;
+ Collection<Pair<Mutation, byte[]>> indexUpdates = new ArrayList<Pair<Mutation, byte[]>>(mutations.size() * 2 * indexMaintainers.size());
+ try {
+ if (!mutableColumns.isEmpty()) {
+ List<KeyRange> keys = Lists.newArrayListWithExpectedSize(mutations.size());
+ for (ImmutableBytesPtr ptr : mutations.keySet()) {
+ keys.add(PVarbinary.INSTANCE.getKeyRange(ptr.copyBytesIfNecessary()));
+ }
+ Scan scan = new Scan();
+ // Project all mutable columns
+ for (ColumnReference ref : mutableColumns) {
+ scan.addColumn(ref.getFamily(), ref.getQualifier());
+ }
+ // Project empty key value column
+ scan.addColumn(indexMaintainers.get(0).getDataEmptyKeyValueCF(), QueryConstants.EMPTY_COLUMN_BYTES);
+ ScanRanges scanRanges = ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, KeyRange.EVERYTHING_RANGE, null, true, -1);
+ scanRanges.initializeScan(scan);
+ scan.setFilter(scanRanges.getSkipScanFilter());
+ TableName tableName = env.getRegion().getRegionInfo().getTable();
+ HTableInterface htable = env.getTable(tableName);
+ txTable = new TransactionAwareHTable(htable);
+ txTable.startTx(tx);
+ // For rollback, we need to see all versions, including
+ // the last committed version as there may be multiple
+ // checkpointed versions.
+ if (isRollback) {
+ tx.setVisibility(VisibilityLevel.SNAPSHOT_ALL);
+ }
+ currentScanner = txTable.getScanner(scan);
+ }
+ if (isRollback) {
+ processRollback(env, indexMetaData, txRollbackAttribute, currentScanner, mutations, tx, mutableColumns, indexUpdates);
+ } else {
+ processMutation(env, indexMetaData, txRollbackAttribute, currentScanner, mutations, tx, mutableColumns, indexUpdates);
+ }
+ } finally {
+ if (txTable != null) txTable.close();
+ }
+
+ return indexUpdates;
+ }
+
+ private void processMutation(RegionCoprocessorEnvironment env,
+ PhoenixIndexMetaData indexMetaData, byte[] txRollbackAttribute,
+ ResultScanner scanner,
+ Map<ImmutableBytesPtr, MultiMutation> mutations, Transaction tx,
+ Set<ColumnReference> mutableColumns,
+ Collection<Pair<Mutation, byte[]>> indexUpdates) throws IOException {
+ if (scanner != null) {
+ Result result;
+ ColumnReference emptyColRef = new ColumnReference(indexMetaData.getIndexMaintainers().get(0).getDataEmptyKeyValueCF(), QueryConstants.EMPTY_COLUMN_BYTES);
+ // Process existing data table rows by removing the old index row and adding the new index row
+ while ((result = scanner.next()) != null) {
+ Mutation m = mutations.remove(new ImmutableBytesPtr(result.getRow()));
+ TxTableState state = new TxTableState(env, mutableColumns, indexMetaData.getAttributes(), tx.getWritePointer(), m, emptyColRef, result);
+ generateDeletes(indexMetaData, indexUpdates, txRollbackAttribute, state);
+ generatePuts(indexMetaData, indexUpdates, state);
+ }
+ }
+ // Process new data table by adding new index rows
+ for (Mutation m : mutations.values()) {
+ TxTableState state = new TxTableState(env, mutableColumns, indexMetaData.getAttributes(), tx.getWritePointer(), m);
+ generatePuts(indexMetaData, indexUpdates, state);
+ }
+ }
+
+ private void processRollback(RegionCoprocessorEnvironment env,
+ PhoenixIndexMetaData indexMetaData, byte[] txRollbackAttribute,
+ ResultScanner scanner,
+ Map<ImmutableBytesPtr, MultiMutation> mutations, Transaction tx,
+ Set<ColumnReference> mutableColumns,
+ Collection<Pair<Mutation, byte[]>> indexUpdates) throws IOException {
+ if (scanner != null) {
+ Result result;
+ // Loop through last committed row state plus all new rows associated with current transaction
+ // to generate point delete markers for all index rows that were added. We don't have Tephra
+ // manage index rows in change sets because we don't want to be hit with the additional
+ // memory hit and do not need to do conflict detection on index rows.
+ ColumnReference emptyColRef = new ColumnReference(indexMetaData.getIndexMaintainers().get(0).getDataEmptyKeyValueCF(), QueryConstants.EMPTY_COLUMN_BYTES);
+ while ((result = scanner.next()) != null) {
+ Mutation m = mutations.remove(new ImmutableBytesPtr(result.getRow()));
+ // Sort by timestamp, type, cf, cq so we can process in time batches from oldest to newest
+ // (as if we're "replaying" them in time order).
+ List<Cell> cells = result.listCells();
+ Collections.sort(cells, new Comparator<Cell>() {
+
+ @Override
+ public int compare(Cell o1, Cell o2) {
+ int c = Longs.compare(o1.getTimestamp(), o2.getTimestamp());
+ if (c != 0) return c;
+ c = o1.getTypeByte() - o2.getTypeByte();
+ if (c != 0) return c;
+ c = Bytes.compareTo(o1.getFamilyArray(), o1.getFamilyOffset(), o1.getFamilyLength(), o1.getFamilyArray(), o1.getFamilyOffset(), o1.getFamilyLength());
+ if (c != 0) return c;
+ return Bytes.compareTo(o1.getQualifierArray(), o1.getQualifierOffset(), o1.getQualifierLength(), o1.getQualifierArray(), o1.getQualifierOffset(), o1.getQualifierLength());
+ }
+
+ });
+ int i = 0;
+ int nCells = cells.size();
+ Result oldResult = null, newResult;
+ long readPtr = tx.getReadPointer();
+ do {
+ boolean hasPuts = false;
+ LinkedList<Cell> singleTimeCells = Lists.newLinkedList();
+ long writePtr;
+ do {
+ Cell cell = cells.get(i);
+ hasPuts |= cell.getTypeByte() == KeyValue.Type.Put.getCode();
+ writePtr = cell.getTimestamp();
+ do {
+ // Add at the beginning of the list to match the expected HBase
+ // newest to oldest sort order (which TxTableState relies on
+ // with the Result.getLatestColumnValue() calls).
+ singleTimeCells.addFirst(cell);
+ } while (++i < nCells && cells.get(i).getTimestamp() == writePtr);
+ } while (i < nCells && cells.get(i).getTimestamp() <= readPtr);
+
+ // Generate point delete markers for the prior row deletion of the old index value.
+ // The write timestamp is the next timestamp, not the current timestamp,
+ // as the earliest cells are the current values for the row (and we don't
+ // want to delete the current row).
+ if (oldResult != null) {
+ TxTableState state = new TxTableState(env, mutableColumns, indexMetaData.getAttributes(), writePtr, m, emptyColRef, oldResult);
+ generateDeletes(indexMetaData, indexUpdates, txRollbackAttribute, state);
+ }
+ // Generate point delete markers for the new index value.
+ // If our time batch doesn't have Puts (i.e. we have only Deletes), then do not
+ // generate deletes. We would have generated the delete above based on the state
+ // of the previous row. The delete markers do not give us the state we need to
+ // delete.
+ if (hasPuts) {
+ newResult = Result.create(singleTimeCells);
+ // First row may represent the current state which we don't want to delete
+ if (writePtr > readPtr) {
+ TxTableState state = new TxTableState(env, mutableColumns, indexMetaData.getAttributes(), writePtr, m, emptyColRef, newResult);
+ generateDeletes(indexMetaData, indexUpdates, txRollbackAttribute, state);
+ }
+ oldResult = newResult;
+ } else {
+ oldResult = null;
+ }
+ } while (i < nCells);
+ }
+ }
+ }
+
+ private void generateDeletes(PhoenixIndexMetaData indexMetaData,
+ Collection<Pair<Mutation, byte[]>> indexUpdates,
+ byte[] attribValue, TxTableState state) throws IOException {
+ Iterable<IndexUpdate> deletes = codec.getIndexDeletes(state, indexMetaData);
+ for (IndexUpdate delete : deletes) {
+ if (delete.isValid()) {
+ delete.getUpdate().setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, attribValue);
+ indexUpdates.add(new Pair<Mutation, byte[]>(delete.getUpdate(),delete.getTableName()));
+ }
+ }
+ }
+
+ private boolean generatePuts(
+ PhoenixIndexMetaData indexMetaData,
+ Collection<Pair<Mutation, byte[]>> indexUpdates,
+ TxTableState state)
+ throws IOException {
+ state.applyMutation();
+ Iterable<IndexUpdate> puts = codec.getIndexUpserts(state, indexMetaData);
+ boolean validPut = false;
+ for (IndexUpdate put : puts) {
+ if (put.isValid()) {
+ indexUpdates.add(new Pair<Mutation, byte[]>(put.getUpdate(),put.getTableName()));
+ validPut = true;
+ }
+ }
+ return validPut;
+ }
+
+
+ private static class TxTableState implements TableState {
+ private final Mutation mutation;
+ private final long currentTimestamp;
+ private final RegionCoprocessorEnvironment env;
+ private final Map<String, byte[]> attributes;
+ private final List<KeyValue> pendingUpdates;
+ private final Set<ColumnReference> indexedColumns;
+ private final Map<ColumnReference, ImmutableBytesWritable> valueMap;
+
+ private TxTableState(RegionCoprocessorEnvironment env, Set<ColumnReference> indexedColumns, Map<String, byte[]> attributes, long currentTimestamp, Mutation mutation) {
+ this.env = env;
+ this.currentTimestamp = currentTimestamp;
+ this.indexedColumns = indexedColumns;
+ this.attributes = attributes;
+ this.mutation = mutation;
+ int estimatedSize = indexedColumns.size();
+ this.valueMap = Maps.newHashMapWithExpectedSize(estimatedSize);
+ this.pendingUpdates = Lists.newArrayListWithExpectedSize(estimatedSize);
+ try {
+ CellScanner scanner = mutation.cellScanner();
+ while (scanner.advance()) {
+ Cell cell = scanner.current();
+ pendingUpdates.add(KeyValueUtil.ensureKeyValue(cell));
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e); // Impossible
+ }
+ }
+
+ public TxTableState(RegionCoprocessorEnvironment env, Set<ColumnReference> indexedColumns, Map<String, byte[]> attributes, long currentTimestamp, Mutation m, ColumnReference emptyColRef, Result r) {
+ this(env, indexedColumns, attributes, currentTimestamp, m);
+
+ for (ColumnReference ref : indexedColumns) {
+ Cell cell = r.getColumnLatestCell(ref.getFamily(), ref.getQualifier());
+ if (cell != null) {
+ ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+ ptr.set(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ valueMap.put(ref, ptr);
+ }
+ }
+ }
+
+ @Override
+ public RegionCoprocessorEnvironment getEnvironment() {
+ return env;
+ }
+
+ @Override
+ public long getCurrentTimestamp() {
+ return currentTimestamp;
+ }
+
+ @Override
+ public Map<String, byte[]> getUpdateAttributes() {
+ return attributes;
+ }
+
+ @Override
+ public byte[] getCurrentRowKey() {
+ return mutation.getRow();
+ }
+
+ @Override
+ public List<? extends IndexedColumnGroup> getIndexColumnHints() {
+ return Collections.emptyList();
+ }
+
+ private void applyMutation() {
+ for (Cell cell : pendingUpdates) {
+ if (cell.getTypeByte() == KeyValue.Type.Delete.getCode() || cell.getTypeByte() == KeyValue.Type.DeleteColumn.getCode()) {
+ ColumnReference ref = new ColumnReference(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(), cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
+ valueMap.remove(ref);
+ } else if (cell.getTypeByte() == KeyValue.Type.DeleteFamily.getCode() || cell.getTypeByte() == KeyValue.Type.DeleteFamilyVersion.getCode()) {
+ for (ColumnReference ref : indexedColumns) {
+ if (ref.matchesFamily(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength())) {
+ valueMap.remove(ref);
+ }
+ }
+ } else if (cell.getTypeByte() == KeyValue.Type.Put.getCode()){
+ ColumnReference ref = new ColumnReference(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(), cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
+ if (indexedColumns.contains(ref)) {
+ ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+ ptr.set(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ valueMap.put(ref, ptr);
+ }
+ } else {
+ throw new IllegalStateException("Unexpected mutation type for " + cell);
+ }
+ }
+ }
+
+ @Override
+ public Collection<KeyValue> getPendingUpdate() {
+ return pendingUpdates;
+ }
+
+ @Override
+ public Pair<ValueGetter, IndexUpdate> getIndexUpdateState(Collection<? extends ColumnReference> indexedColumns)
+ throws IOException {
+ // TODO: creating these objects over and over again is wasteful
+ ColumnTracker tracker = new ColumnTracker(indexedColumns);
+ ValueGetter getter = new ValueGetter() {
+
+ @Override
+ public ImmutableBytesWritable getLatestValue(ColumnReference ref) throws IOException {
+ return valueMap.get(ref);
+ }
+
+ @Override
+ public byte[] getRowKey() {
+ return mutation.getRow();
+ }
+
+ };
+ Pair<ValueGetter, IndexUpdate> pair = new Pair<ValueGetter, IndexUpdate>(getter, new IndexUpdate(tracker));
+ return pair;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 57e84f0..c18233e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -55,6 +55,7 @@ import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
import org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.exception.SQLExceptionInfo;
+import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.filter.ColumnProjectionFilter;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.parse.FilterableStatement;
@@ -104,6 +105,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
private final byte[] physicalTableName;
private final QueryPlan plan;
protected final String scanId;
+ protected final MutationState mutationState;
private final ParallelScanGrouper scanGrouper;
// TODO: too much nesting here - breakup into new classes.
private final List<List<List<Pair<Scan,Future<PeekingResultIterator>>>>> allFutures;
@@ -202,6 +204,9 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
this.plan = plan;
this.scanGrouper = scanGrouper;
StatementContext context = plan.getContext();
+ // Clone MutationState as the one on the connection will change if auto commit is on
+ // yet we need the original one with the original transaction from TableResultIterator.
+ this.mutationState = new MutationState(context.getConnection().getMutationState());
TableRef tableRef = plan.getTableRef();
PTable table = tableRef.getTable();
physicalTableName = table.getPhysicalName().getBytes();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java
index f272e55..ef8c4a6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.compile.StatementContext;
+import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.TableRef;
@@ -51,38 +52,44 @@ public class ChunkedResultIterator implements PeekingResultIterator {
private ImmutableBytesWritable lastKey = new ImmutableBytesWritable();
private final StatementContext context;
private final TableRef tableRef;
- private Scan scan;
private final long chunkSize;
+ private final MutationState mutationState;
+ private Scan scan;
private PeekingResultIterator resultIterator;
public static class ChunkedResultIteratorFactory implements ParallelIteratorFactory {
private final ParallelIteratorFactory delegateFactory;
private final TableRef tableRef;
+ private final MutationState mutationState;
public ChunkedResultIteratorFactory(ParallelIteratorFactory
- delegateFactory, TableRef tableRef) {
+ delegateFactory, MutationState mutationState, TableRef tableRef) {
this.delegateFactory = delegateFactory;
this.tableRef = tableRef;
+ // Clone MutationState, as the one on the connection may change if auto commit is on
+ // while we need a handle to the original one (for it's transaction state).
+ this.mutationState = new MutationState(mutationState);
}
@Override
public PeekingResultIterator newIterator(StatementContext context, ResultIterator scanner, Scan scan, String tableName) throws SQLException {
if (logger.isDebugEnabled()) logger.debug(LogUtil.addCustomAnnotations("ChunkedResultIteratorFactory.newIterator over " + tableRef.getTable().getName().getString() + " with " + scan, ScanUtil.getCustomAnnotations(scan)));
- return new ChunkedResultIterator(delegateFactory, context, tableRef, scan,
- context.getConnection().getQueryServices().getProps().getLong(
- QueryServices.SCAN_RESULT_CHUNK_SIZE,
- QueryServicesOptions.DEFAULT_SCAN_RESULT_CHUNK_SIZE), scanner);
+ return new ChunkedResultIterator(delegateFactory, mutationState, context, tableRef, scan,
+ mutationState.getConnection().getQueryServices().getProps().getLong(
+ QueryServices.SCAN_RESULT_CHUNK_SIZE,
+ QueryServicesOptions.DEFAULT_SCAN_RESULT_CHUNK_SIZE), scanner);
}
}
- private ChunkedResultIterator(ParallelIteratorFactory delegateIteratorFactory,
- StatementContext context, TableRef tableRef, Scan scan, long chunkSize, ResultIterator scanner) throws SQLException {
+ private ChunkedResultIterator(ParallelIteratorFactory delegateIteratorFactory, MutationState mutationState,
+ StatementContext context, TableRef tableRef, Scan scan, long chunkSize, ResultIterator scanner) throws SQLException {
this.delegateIteratorFactory = delegateIteratorFactory;
this.context = context;
this.tableRef = tableRef;
this.scan = scan;
this.chunkSize = chunkSize;
+ this.mutationState = mutationState;
// Instantiate single chunk iterator and the delegate iterator in constructor
// to get parallel scans kicked off in separate threads. If we delay this,
// we'll get serialized behavior (see PHOENIX-
@@ -120,7 +127,7 @@ public class ChunkedResultIterator implements PeekingResultIterator {
if (logger.isDebugEnabled()) logger.debug(LogUtil.addCustomAnnotations("Get next chunked result iterator over " + tableRef.getTable().getName().getString() + " with " + scan, ScanUtil.getCustomAnnotations(scan)));
String tableName = tableRef.getTable().getPhysicalName().getString();
ResultIterator singleChunkResultIterator = new SingleChunkResultIterator(
- new TableResultIterator(context, tableRef, scan, context.getReadMetricsQueue().allotMetric(SCAN_BYTES, tableName)), chunkSize);
+ new TableResultIterator(mutationState, tableRef, scan, context.getReadMetricsQueue().allotMetric(SCAN_BYTES, tableName)), chunkSize);
resultIterator = delegateIteratorFactory.newIterator(context, singleChunkResultIterator, scan, tableName);
}
return resultIterator;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
index 87f8335..265831c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
@@ -102,7 +102,7 @@ public class ParallelIterators extends BaseResultIterators {
@Override
public PeekingResultIterator call() throws Exception {
long startTime = System.currentTimeMillis();
- ResultIterator scanner = new TableResultIterator(context, tableRef, scan, scanMetrics);
+ ResultIterator scanner = new TableResultIterator(mutationState, tableRef, scan, scanMetrics);
if (logger.isDebugEnabled()) {
logger.debug(LogUtil.addCustomAnnotations("Id: " + scanId + ", Time: " + (System.currentTimeMillis() - startTime) + "ms, Scan: " + scan, ScanUtil.getCustomAnnotations(scan)));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
index fa18c83..e0936b2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
@@ -80,7 +80,7 @@ public class SerialIterators extends BaseResultIterators {
public PeekingResultIterator call() throws Exception {
List<PeekingResultIterator> concatIterators = Lists.newArrayListWithExpectedSize(scans.size());
for (final Scan scan : scans) {
- ResultIterator scanner = new TableResultIterator(context, tableRef, scan, context.getReadMetricsQueue().allotMetric(SCAN_BYTES, tableName), ScannerCreation.DELAYED);
+ ResultIterator scanner = new TableResultIterator(mutationState, tableRef, scan, context.getReadMetricsQueue().allotMetric(SCAN_BYTES, tableName), ScannerCreation.DELAYED);
concatIterators.add(iteratorFactory.newIterator(context, scanner, scan, tableName));
}
PeekingResultIterator concatIterator = ConcatResultIterator.newIterator(concatIterators);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java
index 6f040d1..7d07a4a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java
@@ -22,8 +22,11 @@ import java.sql.SQLException;
import java.util.List;
import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.phoenix.compile.StatementContext;
+import org.apache.phoenix.execute.MutationState;
+import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.monitoring.CombinableMetric;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.tuple.Tuple;
@@ -39,7 +42,7 @@ import org.apache.phoenix.util.ServerUtil;
*
* @since 0.1
*/
-public class TableResultIterator extends ExplainTable implements ResultIterator {
+public class TableResultIterator implements ResultIterator {
public enum ScannerCreation {IMMEDIATE, DELAYED};
private final Scan scan;
@@ -47,8 +50,8 @@ public class TableResultIterator extends ExplainTable implements ResultIterator
private volatile ResultIterator delegate;
private final CombinableMetric scanMetrics;
- public TableResultIterator(StatementContext context, TableRef tableRef, CombinableMetric scanMetrics) throws SQLException {
- this(context, tableRef, context.getScan(), scanMetrics);
+ public TableResultIterator(MutationState mutationState, Scan scan, TableRef tableRef, CombinableMetric scanMetrics) throws SQLException {
+ this(mutationState, tableRef, scan, scanMetrics);
}
/*
@@ -75,15 +78,15 @@ public class TableResultIterator extends ExplainTable implements ResultIterator
return delegate;
}
- public TableResultIterator(StatementContext context, TableRef tableRef, Scan scan, CombinableMetric scanMetrics) throws SQLException {
- this(context, tableRef, scan, scanMetrics, ScannerCreation.IMMEDIATE);
+ public TableResultIterator(MutationState mutationState, TableRef tableRef, Scan scan, CombinableMetric scanMetrics) throws SQLException {
+ this(mutationState, tableRef, scan, scanMetrics, ScannerCreation.IMMEDIATE);
}
- public TableResultIterator(StatementContext context, TableRef tableRef, Scan scan, CombinableMetric scanMetrics, ScannerCreation creationMode) throws SQLException {
- super(context, tableRef);
+ public TableResultIterator(MutationState mutationState, TableRef tableRef, Scan scan, CombinableMetric scanMetrics, ScannerCreation creationMode) throws SQLException {
this.scan = scan;
this.scanMetrics = scanMetrics;
- htable = context.getConnection().getQueryServices().getTable(tableRef.getTable().getPhysicalName().getBytes());
+ PTable table = tableRef.getTable();
+ htable = mutationState.getHTable(table);
if (creationMode == ScannerCreation.IMMEDIATE) {
getDelegate(false);
}
@@ -109,8 +112,11 @@ public class TableResultIterator extends ExplainTable implements ResultIterator
@Override
public void explain(List<String> planSteps) {
- StringBuilder buf = new StringBuilder();
- explain(buf.toString(),planSteps);
+ try {
+ getDelegate(false).explain(planSteps);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
index 6a142e5..e864bfe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
@@ -79,6 +79,7 @@ import org.apache.phoenix.schema.PMetaData.Pruner;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.PTableRef;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.schema.types.PArrayDataType;
@@ -158,14 +159,18 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
}
public PhoenixConnection(PhoenixConnection connection, boolean isDescRowKeyOrderUpgrade) throws SQLException {
- this(connection.getQueryServices(), connection.getURL(), connection.getClientInfo(), connection.getMetaDataCache(), isDescRowKeyOrderUpgrade);
+ this(connection.getQueryServices(), connection.getURL(), connection.getClientInfo(), connection.metaData, connection.getMutationState(), isDescRowKeyOrderUpgrade);
this.isAutoCommit = connection.isAutoCommit;
this.sampler = connection.sampler;
this.statementExecutionCounter = connection.statementExecutionCounter;
}
public PhoenixConnection(PhoenixConnection connection) throws SQLException {
- this(connection, connection.isDescVarLengthRowKeyUpgrade);
+ this(connection, connection.isDescVarLengthRowKeyUpgrade());
+ }
+
+ public PhoenixConnection(PhoenixConnection connection, MutationState mutationState) throws SQLException {
+ this(connection.getQueryServices(), connection.getURL(), connection.getClientInfo(), connection.getMetaDataCache(), mutationState, connection.isDescVarLengthRowKeyUpgrade());
}
public PhoenixConnection(PhoenixConnection connection, long scn) throws SQLException {
@@ -173,17 +178,21 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
}
public PhoenixConnection(ConnectionQueryServices services, PhoenixConnection connection, long scn) throws SQLException {
- this(services, connection.getURL(), newPropsWithSCN(scn,connection.getClientInfo()), connection.getMetaDataCache(), connection.isDescVarLengthRowKeyUpgrade());
+ this(services, connection.getURL(), newPropsWithSCN(scn,connection.getClientInfo()), connection.metaData, connection.getMutationState(), connection.isDescVarLengthRowKeyUpgrade());
this.isAutoCommit = connection.isAutoCommit;
this.sampler = connection.sampler;
this.statementExecutionCounter = connection.statementExecutionCounter;
}
public PhoenixConnection(ConnectionQueryServices services, String url, Properties info, PMetaData metaData) throws SQLException {
- this(services, url, info, metaData, false);
+ this(services, url, info, metaData, null, false);
}
-
- public PhoenixConnection(ConnectionQueryServices services, String url, Properties info, PMetaData metaData, boolean isDescVarLengthRowKeyUpgrade) throws SQLException {
+
+ public PhoenixConnection(PhoenixConnection connection, ConnectionQueryServices services, Properties info) throws SQLException {
+ this(services, connection.url, info, connection.metaData, null, connection.isDescVarLengthRowKeyUpgrade());
+ }
+
+ public PhoenixConnection(ConnectionQueryServices services, String url, Properties info, PMetaData metaData, MutationState mutationState, boolean isDescVarLengthRowKeyUpgrade) throws SQLException {
this.url = url;
this.isDescVarLengthRowKeyUpgrade = isDescVarLengthRowKeyUpgrade;
// Copy so client cannot change
@@ -255,7 +264,7 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
}
};
this.isRequestLevelMetricsEnabled = JDBCUtil.isCollectingRequestLevelMetricsEnabled(url, info, this.services.getProps());
- this.mutationState = newMutationState(maxSize);
+ this.mutationState = mutationState == null ? newMutationState(maxSize) : new MutationState(mutationState);
this.metaData = metaData.pruneTables(pruner);
this.metaData = metaData.pruneFunctions(pruner);
this.services.addConnection(this);
@@ -408,12 +417,16 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
return mutateBatchSize;
}
+ public PMetaData getMetaDataCache() {
+ return metaData;
+ }
+
public PTable getTable(PTableKey key) throws TableNotFoundException {
- return metaData.getTable(key);
+ return metaData.getTableRef(key).getTable();
}
- public PMetaData getMetaDataCache() {
- return metaData;
+ public PTableRef getTableRef(PTableKey key) throws TableNotFoundException {
+ return metaData.getTableRef(key);
}
protected MutationState newMutationState(int maxSize) {
@@ -450,7 +463,9 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
// from modifying this list.
this.statements = Lists.newArrayList();
try {
- mutationState.rollback(this);
+ mutationState.rollback();
+ } catch (SQLException e) {
+ // ignore any exceptions while rolling back
} finally {
try {
SQLCloseables.closeAll(statements);
@@ -693,7 +708,13 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
@Override
public void rollback() throws SQLException {
- mutationState.rollback(this);
+ CallRunner.run(new CallRunner.CallableThrowable<Void, SQLException>() {
+ @Override
+ public Void call() throws SQLException {
+ mutationState.rollback();
+ return null;
+ }
+ }, Tracing.withTracing(this, "rolling back"));
statementExecutionCounter = 0;
}
@@ -806,16 +827,20 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
// TODO Auto-generated method stub
return 0;
}
-
+
@Override
- public PMetaData addTable(PTable table) throws SQLException {
- // TODO: since a connection is only used by one thread at a time,
- // we could modify this metadata in place since it's not shared.
- if (scn == null || scn > table.getTimeStamp()) {
- metaData = metaData.addTable(table);
- }
+ public PMetaData addTable(PTable table, long resolvedTime) throws SQLException {
+ metaData = metaData.addTable(table, resolvedTime);
//Cascade through to connectionQueryServices too
- getQueryServices().addTable(table);
+ getQueryServices().addTable(table, resolvedTime);
+ return metaData;
+ }
+
+ @Override
+ public PMetaData updateResolvedTimestamp(PTable table, long resolvedTime) throws SQLException {
+ metaData = metaData.updateResolvedTimestamp(table, resolvedTime);
+ //Cascade through to connectionQueryServices too
+ getQueryServices().updateResolvedTimestamp(table, resolvedTime);
return metaData;
}
@@ -832,11 +857,11 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
}
@Override
- public PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls)
+ public PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime)
throws SQLException {
- metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls);
+ metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime);
//Cascade through to connectionQueryServices too
- getQueryServices().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls);
+ getQueryServices().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime);
return metaData;
}
@@ -858,10 +883,10 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
@Override
public PMetaData removeColumn(PName tenantId, String tableName, List<PColumn> columnsToRemove, long tableTimeStamp,
- long tableSeqNum) throws SQLException {
- metaData = metaData.removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum);
+ long tableSeqNum, long resolvedTime) throws SQLException {
+ metaData = metaData.removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum, resolvedTime);
//Cascade through to connectionQueryServices too
- getQueryServices().removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum);
+ getQueryServices().removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum, resolvedTime);
return metaData;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index 3f6958c..d6d1230 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -301,6 +301,9 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData, org.apache.pho
private static final String TENANT_POS_SHIFT = "TENANT_POS_SHIFT";
private static final byte[] TENANT_POS_SHIFT_BYTES = Bytes.toBytes(TENANT_POS_SHIFT);
+ public static final String TRANSACTIONAL = "TRANSACTIONAL";
+ public static final byte[] TRANSACTIONAL_BYTES = Bytes.toBytes(TRANSACTIONAL);
+
private final PhoenixConnection connection;
private final ResultSet emptyResultSet;
public static final int MAX_LOCAL_SI_VERSION_DISALLOW = VersionUtil.encodeVersion("0", "98", "8");
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
index 1a785a6..bb0b1e2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
@@ -162,7 +162,7 @@ public final class PhoenixDriver extends PhoenixEmbeddedDriver {
ConnectionQueryServices connectionQueryServices = connectionQueryServicesMap.get(normalizedConnInfo);
if (connectionQueryServices == null) {
if (normalizedConnInfo.isConnectionless()) {
- connectionQueryServices = new ConnectionlessQueryServicesImpl(services, normalizedConnInfo);
+ connectionQueryServices = new ConnectionlessQueryServicesImpl(services, normalizedConnInfo, info);
} else {
connectionQueryServices = new ConnectionQueryServicesImpl(services, normalizedConnInfo, info);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java
index 3cfaacc..8ee9dbd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java
@@ -187,6 +187,10 @@ public abstract class PhoenixEmbeddedDriver implements Driver, org.apache.phoeni
.setMessage(url).build().buildException();
}
+ public String getZookeeperConnectionString() {
+ return getZookeeperQuorum() + ":" + getPort();
+ }
+
/**
* Detect url with quorum:1,quorum:2 as HBase does not handle different port numbers
* for different quorum hostnames.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index f274336..bacea92 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -34,6 +34,7 @@ import java.text.Format;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -263,6 +264,12 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho
final long startTime = System.currentTimeMillis();
try {
QueryPlan plan = stmt.compilePlan(PhoenixStatement.this, Sequence.ValueOp.VALIDATE_SEQUENCE);
+ // Send mutations to hbase, so they are visible to subsequent reads.
+ // Use original plan for data table so that data and immutable indexes will be sent
+ // TODO: for joins, we need to iterate through all tables, but we need the original table,
+ // not the projected table, so plan.getContext().getResolver().getTables() won't work.
+ Iterator<TableRef> tableRefs = plan.getSourceRefs().iterator();
+ connection.getMutationState().sendUncommitted(tableRefs);
plan = connection.getQueryServices().getOptimizer().optimize(PhoenixStatement.this, plan);
// this will create its own trace internally, so we don't wrap this
// whole thing in tracing
@@ -322,6 +329,12 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho
try {
MutationState state = connection.getMutationState();
MutationPlan plan = stmt.compilePlan(PhoenixStatement.this, Sequence.ValueOp.VALIDATE_SEQUENCE);
+ if (plan.getTargetRef() != null && plan.getTargetRef().getTable() != null && plan.getTargetRef().getTable().isTransactional()) {
+ state.startTransaction();
+ }
+ Iterator<TableRef> tableRefs = plan.getSourceRefs().iterator();
+ state.sendUncommitted(tableRefs);
+ state.checkpointIfNeccessary(plan);
MutationState lastState = plan.execute();
state.join(lastState);
if (connection.getAutoCommit()) {
@@ -1608,4 +1621,4 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
index f13b28e..e8b3389 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
@@ -39,6 +39,7 @@ import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.ServerUtil;
import org.apache.phoenix.util.TrustedByteArrayOutputStream;
import org.apache.phoenix.util.TupleUtil;
@@ -80,7 +81,7 @@ public class HashCacheClient {
*/
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
serialize(ptr, iterator, estimatedSize, onExpressions, singleValueOnly, keyRangeRhsExpression, keyRangeRhsValues);
- return serverCache.addServerCache(keyRanges, ptr, new HashCacheFactory(), cacheUsingTableRef);
+ return serverCache.addServerCache(keyRanges, ptr, ByteUtil.EMPTY_BYTE_ARRAY, new HashCacheFactory(), cacheUsingTableRef);
}
private void serialize(ImmutableBytesWritable ptr, ResultIterator iterator, long estimatedSize, List<Expression> onExpressions, boolean singleValueOnly, Expression keyRangeRhsExpression, List<Expression> keyRangeRhsValues) throws SQLException {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
index 8cae51a..3072736 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
@@ -56,7 +56,7 @@ public class HashCacheFactory implements ServerCacheFactory {
}
@Override
- public Closeable newCache(ImmutableBytesWritable cachePtr, MemoryChunk chunk) throws SQLException {
+ public Closeable newCache(ImmutableBytesWritable cachePtr, byte[] txState, MemoryChunk chunk) throws SQLException {
try {
// This reads the uncompressed length from the front of the compressed input
int uncompressedLen = Snappy.getUncompressedLength(cachePtr.get(), cachePtr.getOffset());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java
index b500a25..d2f91b3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java
@@ -108,7 +108,7 @@ public class PhoenixRecordReader<T extends DBWritable> extends RecordReader<Null
ReadMetricQueue readMetrics = ctx.getReadMetricsQueue();
String tableName = queryPlan.getTableRef().getTable().getPhysicalName().getString();
for (Scan scan : scans) {
- final TableResultIterator tableResultIterator = new TableResultIterator(queryPlan.getContext(),
+ final TableResultIterator tableResultIterator = new TableResultIterator(queryPlan.getContext().getConnection().getMutationState(),
queryPlan.getTableRef(), scan, readMetrics.allotMetric(SCAN_BYTES, tableName));
PeekingResultIterator peekingResultIterator = LookAheadResultIterator.wrap(tableResultIterator);
iterators.add(peekingResultIterator);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectMapper.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectMapper.java
index 32b66f1..3bc3808 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectMapper.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectMapper.java
@@ -138,7 +138,7 @@ public class PhoenixIndexImportDirectMapper extends
private void writeBatch(MutationState mutationState, Context context) throws IOException,
SQLException, InterruptedException {
- final Iterator<Pair<byte[], List<Mutation>>> iterator = mutationState.toMutations(true);
+ final Iterator<Pair<byte[], List<Mutation>>> iterator = mutationState.toMutations(true, null);
while (iterator.hasNext()) {
Pair<byte[], List<Mutation>> mutationPair = iterator.next();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
index bc318d2..c5ff3c6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
@@ -95,17 +95,11 @@ public class QueryOptimizer {
return plans.get(0);
}
- public List<QueryPlan> getBestPlan(PhoenixStatement statement, SelectStatement select, ColumnResolver resolver, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory) throws SQLException {
- return getApplicablePlans(statement, select, resolver, targetColumns, parallelIteratorFactory, true);
+ public List<QueryPlan> getBestPlan(QueryPlan dataPlan, PhoenixStatement statement, SelectStatement select, ColumnResolver resolver, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory) throws SQLException {
+ return getApplicablePlans(dataPlan, statement, targetColumns, parallelIteratorFactory, true);
}
- public List<QueryPlan> getApplicablePlans(PhoenixStatement statement, SelectStatement select, ColumnResolver resolver, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory) throws SQLException {
- return getApplicablePlans(statement, select, resolver, targetColumns, parallelIteratorFactory, false);
- }
-
- private List<QueryPlan> getApplicablePlans(PhoenixStatement statement, SelectStatement select, ColumnResolver resolver, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory, boolean stopAtBestPlan) throws SQLException {
- QueryCompiler compiler = new QueryCompiler(statement, select, resolver, targetColumns, parallelIteratorFactory, new SequenceManager(statement));
- QueryPlan dataPlan = compiler.compile();
+ public List<QueryPlan> getApplicablePlans(QueryPlan dataPlan, PhoenixStatement statement, SelectStatement select, ColumnResolver resolver, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory) throws SQLException {
return getApplicablePlans(dataPlan, statement, targetColumns, parallelIteratorFactory, false);
}
@@ -221,7 +215,8 @@ public class QueryOptimizer {
private static QueryPlan addPlan(PhoenixStatement statement, SelectStatement select, PTable index, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory, QueryPlan dataPlan, boolean isHinted) throws SQLException {
int nColumns = dataPlan.getProjector().getColumnCount();
- String alias = '"' + dataPlan.getTableRef().getTableAlias() + '"'; // double quote in case it's case sensitive
+ String tableAlias = dataPlan.getTableRef().getTableAlias();
+ String alias = tableAlias==null ? null : '"' + tableAlias + '"'; // double quote in case it's case sensitive
String schemaName = index.getParentSchemaName().getString();
schemaName = schemaName.length() == 0 ? null : '"' + schemaName + '"';
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index 3b53309..08c417f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -44,6 +44,8 @@ import org.apache.phoenix.schema.SequenceAllocation;
import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.stats.PTableStats;
+import co.cask.tephra.TransactionSystemClient;
+
public interface ConnectionQueryServices extends QueryServices, MetaDataMutated {
public static final int INITIAL_META_DATA_TABLE_CAPACITY = 100;
@@ -88,7 +90,7 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
void clearTableRegionCache(byte[] tableName) throws SQLException;
- boolean hasInvalidIndexConfiguration();
+ boolean isMutableIndexWALCodecInstalled();
long createSequence(String tenantId, String schemaName, String sequenceName, long startWith, long incrementBy, long cacheSize, long minValue, long maxValue, boolean cycle, long timestamp) throws SQLException;
long dropSequence(String tenantId, String schemaName, String sequenceName, long timestamp) throws SQLException;
@@ -116,4 +118,6 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
public void clearCache() throws SQLException;
public int getSequenceSaltBuckets();
+
+ TransactionSystemClient getTransactionSystemClient();
}
\ No newline at end of file
[7/9] phoenix git commit: PHOENIX-1674 Snapshot isolation transaction
support through Tephra (James Taylor, Thomas D'Silva)
Posted by td...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
new file mode 100644
index 0000000..6bcf7ea
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
@@ -0,0 +1,537 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.tx;
+
+import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.apache.phoenix.util.TestUtil.TRANSACTIONAL_DATA_TABLE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.Before;
+import org.junit.Test;
+
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.hbase11.coprocessor.TransactionProcessor;
+
+import com.google.common.collect.Lists;
+
+public class TransactionIT extends BaseHBaseManagedTimeIT {
+
+ private static final String FULL_TABLE_NAME = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + TRANSACTIONAL_DATA_TABLE;
+
+ @Before
+ public void setUp() throws SQLException {
+ ensureTableCreated(getUrl(), TRANSACTIONAL_DATA_TABLE);
+ }
+
+ @Test
+ public void testReadOwnWrites() throws Exception {
+ String selectSql = "SELECT * FROM "+FULL_TABLE_NAME;
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(false);
+ ResultSet rs = conn.createStatement().executeQuery(selectSql);
+ assertFalse(rs.next());
+
+ String upsert = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn.prepareStatement(upsert);
+ // upsert two rows
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.execute();
+ TestUtil.setRowKeyColumns(stmt, 2);
+ stmt.execute();
+
+ // verify rows can be read even though commit has not been called
+ rs = conn.createStatement().executeQuery(selectSql);
+ TestUtil.validateRowKeyColumns(rs, 1);
+ TestUtil.validateRowKeyColumns(rs, 2);
+ assertFalse(rs.next());
+
+ conn.commit();
+
+ // verify rows can be read after commit
+ rs = conn.createStatement().executeQuery(selectSql);
+ TestUtil.validateRowKeyColumns(rs, 1);
+ TestUtil.validateRowKeyColumns(rs, 2);
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testTxnClosedCorrecty() throws Exception {
+ String selectSql = "SELECT * FROM "+FULL_TABLE_NAME;
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(false);
+ ResultSet rs = conn.createStatement().executeQuery(selectSql);
+ assertFalse(rs.next());
+
+ String upsert = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn.prepareStatement(upsert);
+ // upsert two rows
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.execute();
+ TestUtil.setRowKeyColumns(stmt, 2);
+ stmt.execute();
+
+ // verify rows can be read even though commit has not been called
+ rs = conn.createStatement().executeQuery(selectSql);
+ TestUtil.validateRowKeyColumns(rs, 1);
+ TestUtil.validateRowKeyColumns(rs, 2);
+ assertFalse(rs.next());
+
+ conn.close();
+ // wait for any open txns to time out
+ Thread.sleep(DEFAULT_TXN_TIMEOUT_SECONDS*1000+10000);
+ assertTrue("There should be no invalid transactions", txManager.getInvalidSize()==0);
+ }
+ }
+
+ @Test
+ public void testDelete() throws Exception {
+ String selectSQL = "SELECT * FROM " + FULL_TABLE_NAME;
+ try (Connection conn1 = DriverManager.getConnection(getUrl());
+ Connection conn2 = DriverManager.getConnection(getUrl())) {
+ conn1.setAutoCommit(false);
+ ResultSet rs = conn1.createStatement().executeQuery(selectSQL);
+ assertFalse(rs.next());
+
+ String upsert = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn1.prepareStatement(upsert);
+ // upsert two rows
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.execute();
+ conn1.commit();
+
+ TestUtil.setRowKeyColumns(stmt, 2);
+ stmt.execute();
+
+ // verify rows can be read even though commit has not been called
+ int rowsDeleted = conn1.createStatement().executeUpdate("DELETE FROM " + FULL_TABLE_NAME);
+ assertEquals(2, rowsDeleted);
+
+ // Delete and second upsert not committed yet, so there should be one row.
+ rs = conn2.createStatement().executeQuery("SELECT count(*) FROM " + FULL_TABLE_NAME);
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+
+ conn1.commit();
+
+ // verify rows are deleted after commit
+ rs = conn1.createStatement().executeQuery(selectSQL);
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testAutoCommitQuerySingleTable() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(true);
+ // verify no rows returned
+ ResultSet rs = conn.createStatement().executeQuery("SELECT * FROM " + FULL_TABLE_NAME);
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testAutoCommitQueryMultiTables() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(true);
+ // verify no rows returned
+ ResultSet rs = conn.createStatement().executeQuery("SELECT * FROM " + FULL_TABLE_NAME + " a JOIN " + FULL_TABLE_NAME + " b ON (a.long_pk = b.int_pk)");
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testColConflicts() throws Exception {
+ try (Connection conn1 = DriverManager.getConnection(getUrl());
+ Connection conn2 = DriverManager.getConnection(getUrl())) {
+ conn1.setAutoCommit(false);
+ conn2.setAutoCommit(false);
+ String selectSql = "SELECT * FROM "+FULL_TABLE_NAME;
+ conn1.setAutoCommit(false);
+ ResultSet rs = conn1.createStatement().executeQuery(selectSql);
+ assertFalse(rs.next());
+ // upsert row using conn1
+ String upsertSql = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk, a.int_col1) VALUES(?, ?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn1.prepareStatement(upsertSql);
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.setInt(7, 10);
+ stmt.execute();
+ // upsert row using conn2
+ stmt = conn2.prepareStatement(upsertSql);
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.setInt(7, 11);
+ stmt.execute();
+
+ conn1.commit();
+ //second commit should fail
+ try {
+ conn2.commit();
+ fail();
+ }
+ catch (SQLException e) {
+ assertEquals(e.getErrorCode(), SQLExceptionCode.TRANSACTION_CONFLICT_EXCEPTION.getErrorCode());
+ }
+ }
+ }
+
+ private void testRowConflicts() throws Exception {
+ try (Connection conn1 = DriverManager.getConnection(getUrl());
+ Connection conn2 = DriverManager.getConnection(getUrl())) {
+ conn1.setAutoCommit(false);
+ conn2.setAutoCommit(false);
+ String selectSql = "SELECT * FROM "+FULL_TABLE_NAME;
+ conn1.setAutoCommit(false);
+ ResultSet rs = conn1.createStatement().executeQuery(selectSql);
+ boolean immutableRows = conn1.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, FULL_TABLE_NAME)).isImmutableRows();
+ assertFalse(rs.next());
+ // upsert row using conn1
+ String upsertSql = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk, a.int_col1) VALUES(?, ?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn1.prepareStatement(upsertSql);
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.setInt(7, 10);
+ stmt.execute();
+ // upsert row using conn2
+ upsertSql = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk, b.int_col2) VALUES(?, ?, ?, ?, ?, ?, ?)";
+ stmt = conn2.prepareStatement(upsertSql);
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.setInt(7, 11);
+ stmt.execute();
+
+ conn1.commit();
+ //second commit should fail
+ try {
+ conn2.commit();
+ if (!immutableRows) fail();
+ }
+ catch (SQLException e) {
+ if (immutableRows) fail();
+ assertEquals(e.getErrorCode(), SQLExceptionCode.TRANSACTION_CONFLICT_EXCEPTION.getErrorCode());
+ }
+ }
+ }
+
+ @Test
+ public void testRowConflictDetected() throws Exception {
+ testRowConflicts();
+ }
+
+ @Test
+ public void testNoConflictDetectionForImmutableRows() throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ conn.createStatement().execute("ALTER TABLE " + FULL_TABLE_NAME + " SET IMMUTABLE_ROWS=true");
+ testRowConflicts();
+ }
+
+ @Test
+ public void testNonTxToTxTable() throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ conn.createStatement().execute("CREATE TABLE NON_TX_TABLE(k INTEGER PRIMARY KEY, v VARCHAR)");
+ conn.createStatement().execute("UPSERT INTO NON_TX_TABLE VALUES (1)");
+ conn.createStatement().execute("UPSERT INTO NON_TX_TABLE VALUES (2, 'a')");
+ conn.createStatement().execute("UPSERT INTO NON_TX_TABLE VALUES (3, 'b')");
+ conn.commit();
+
+ conn.createStatement().execute("CREATE INDEX IDX ON NON_TX_TABLE(v)");
+ // Reset empty column value to an empty value like it is pre-transactions
+ HTableInterface htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("NON_TX_TABLE"));
+ List<Put>puts = Lists.newArrayList(new Put(PInteger.INSTANCE.toBytes(1)), new Put(PInteger.INSTANCE.toBytes(2)), new Put(PInteger.INSTANCE.toBytes(3)));
+ for (Put put : puts) {
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, ByteUtil.EMPTY_BYTE_ARRAY);
+ }
+ htable.put(puts);
+
+ conn.createStatement().execute("ALTER TABLE NON_TX_TABLE SET TRANSACTIONAL=true");
+
+ htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("NON_TX_TABLE"));
+ assertTrue(htable.getTableDescriptor().getCoprocessors().contains(TransactionProcessor.class.getName()));
+ htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("IDX"));
+ assertTrue(htable.getTableDescriptor().getCoprocessors().contains(TransactionProcessor.class.getName()));
+
+ conn.createStatement().execute("UPSERT INTO NON_TX_TABLE VALUES (4, 'c')");
+ ResultSet rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ k FROM NON_TX_TABLE WHERE v IS NULL");
+ assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "NON_TX_TABLE")).isTransactional());
+ assertTrue(rs.next());
+ assertEquals(1,rs.getInt(1));
+ assertFalse(rs.next());
+ conn.commit();
+
+ conn.createStatement().execute("UPSERT INTO NON_TX_TABLE VALUES (5, 'd')");
+ rs = conn.createStatement().executeQuery("SELECT k FROM NON_TX_TABLE");
+ assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "IDX")).isTransactional());
+ assertTrue(rs.next());
+ assertEquals(1,rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(2,rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(3,rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(4,rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(5,rs.getInt(1));
+ assertFalse(rs.next());
+ conn.rollback();
+
+ rs = conn.createStatement().executeQuery("SELECT k FROM NON_TX_TABLE");
+ assertTrue(rs.next());
+ assertEquals(1,rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(2,rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(3,rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(4,rs.getInt(1));
+ assertFalse(rs.next());
+ }
+
+ @Test
+ public void testNonTxToTxTableFailure() throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ // Put table in SYSTEM schema to prevent attempts to update the cache after we disable SYSTEM.CATALOG
+ conn.createStatement().execute("CREATE TABLE SYSTEM.NON_TX_TABLE(k INTEGER PRIMARY KEY, v VARCHAR)");
+ conn.createStatement().execute("UPSERT INTO SYSTEM.NON_TX_TABLE VALUES (1)");
+ conn.commit();
+ // Reset empty column value to an empty value like it is pre-transactions
+ HTableInterface htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("SYSTEM.NON_TX_TABLE"));
+ Put put = new Put(PInteger.INSTANCE.toBytes(1));
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, ByteUtil.EMPTY_BYTE_ARRAY);
+ htable.put(put);
+
+ HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
+ admin.disableTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME);
+ try {
+ // This will succeed initially in updating the HBase metadata, but then will fail when
+ // the SYSTEM.CATALOG table is attempted to be updated, exercising the code to restore
+ // the coprocessors back to the non transactional ones.
+ conn.createStatement().execute("ALTER TABLE SYSTEM.NON_TX_TABLE SET TRANSACTIONAL=true");
+ fail();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME + " is disabled"));
+ } finally {
+ admin.enableTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME);
+ admin.close();
+ }
+
+ ResultSet rs = conn.createStatement().executeQuery("SELECT k FROM SYSTEM.NON_TX_TABLE WHERE v IS NULL");
+ assertTrue(rs.next());
+ assertEquals(1,rs.getInt(1));
+ assertFalse(rs.next());
+
+ htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("SYSTEM.NON_TX_TABLE"));
+ assertFalse(htable.getTableDescriptor().getCoprocessors().contains(TransactionProcessor.class.getName()));
+ assertEquals(1,conn.unwrap(PhoenixConnection.class).getQueryServices().
+ getTableDescriptor(Bytes.toBytes("SYSTEM.NON_TX_TABLE")).
+ getFamily(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES).getMaxVersions());
+ }
+
+ @Test
+ public void testProperties() throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ conn.createStatement().execute("CREATE TABLE NON_TX_TABLE1(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR) TTL=1000");
+ conn.createStatement().execute("CREATE INDEX idx1 ON NON_TX_TABLE1(a.v, b.v) TTL=1000");
+ conn.createStatement().execute("CREATE INDEX idx2 ON NON_TX_TABLE1(c.v) INCLUDE (a.v, b.v) TTL=1000");
+
+ conn.createStatement().execute("ALTER TABLE NON_TX_TABLE1 SET TRANSACTIONAL=true");
+
+ HTableDescriptor desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("NON_TX_TABLE1"));
+ for (HColumnDescriptor colDesc : desc.getFamilies()) {
+ assertEquals(QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
+ assertEquals(1000, colDesc.getTimeToLive());
+ assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
+ }
+
+ desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("IDX1"));
+ for (HColumnDescriptor colDesc : desc.getFamilies()) {
+ assertEquals(QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
+ assertEquals(1000, colDesc.getTimeToLive());
+ assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
+ }
+
+ desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("IDX2"));
+ for (HColumnDescriptor colDesc : desc.getFamilies()) {
+ assertEquals(QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
+ assertEquals(1000, colDesc.getTimeToLive());
+ assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
+ }
+
+ conn.createStatement().execute("CREATE TABLE NON_TX_TABLE2(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)");
+ conn.createStatement().execute("ALTER TABLE NON_TX_TABLE2 SET TRANSACTIONAL=true, VERSIONS=10");
+ desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("NON_TX_TABLE2"));
+ for (HColumnDescriptor colDesc : desc.getFamilies()) {
+ assertEquals(10, colDesc.getMaxVersions());
+ assertEquals(HColumnDescriptor.DEFAULT_TTL, colDesc.getTimeToLive());
+ assertEquals(null, colDesc.getValue(TxConstants.PROPERTY_TTL));
+ }
+ conn.createStatement().execute("ALTER TABLE NON_TX_TABLE2 SET TTL=1000");
+ desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("NON_TX_TABLE2"));
+ for (HColumnDescriptor colDesc : desc.getFamilies()) {
+ assertEquals(10, colDesc.getMaxVersions());
+ assertEquals(1000, colDesc.getTimeToLive());
+ assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
+ }
+
+ conn.createStatement().execute("CREATE TABLE NON_TX_TABLE3(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)");
+ conn.createStatement().execute("ALTER TABLE NON_TX_TABLE3 SET TRANSACTIONAL=true, b.VERSIONS=10, c.VERSIONS=20");
+ desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("NON_TX_TABLE3"));
+ assertEquals(QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, desc.getFamily(Bytes.toBytes("A")).getMaxVersions());
+ assertEquals(10, desc.getFamily(Bytes.toBytes("B")).getMaxVersions());
+ assertEquals(20, desc.getFamily(Bytes.toBytes("C")).getMaxVersions());
+
+ conn.createStatement().execute("CREATE TABLE NON_TX_TABLE4(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)");
+ try {
+ conn.createStatement().execute("ALTER TABLE NON_TX_TABLE4 SET TRANSACTIONAL=true, VERSIONS=1");
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE.getErrorCode(), e.getErrorCode());
+ }
+
+ try {
+ conn.createStatement().execute("ALTER TABLE NON_TX_TABLE4 SET TRANSACTIONAL=true, b.VERSIONS=1");
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE.getErrorCode(), e.getErrorCode());
+ }
+
+ conn.createStatement().execute("CREATE TABLE TX_TABLE1(k INTEGER PRIMARY KEY, v VARCHAR) TTL=1000, TRANSACTIONAL=true");
+ desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("TX_TABLE1"));
+ for (HColumnDescriptor colDesc : desc.getFamilies()) {
+ assertEquals(QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
+ assertEquals(HColumnDescriptor.DEFAULT_TTL, colDesc.getTimeToLive());
+ assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
+ }
+ }
+
+ @Test
+ public void testCreateTableToBeTransactional() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String ddl = "CREATE TABLE TEST_TRANSACTIONAL_TABLE (k varchar primary key) transactional=true";
+ conn.createStatement().execute(ddl);
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ PTable table = pconn.getTable(new PTableKey(null, "TEST_TRANSACTIONAL_TABLE"));
+ HTableInterface htable = pconn.getQueryServices().getTable(Bytes.toBytes("TEST_TRANSACTIONAL_TABLE"));
+ assertTrue(table.isTransactional());
+ assertTrue(htable.getTableDescriptor().getCoprocessors().contains(TransactionProcessor.class.getName()));
+
+ try {
+ ddl = "ALTER TABLE TEST_TRANSACTIONAL_TABLE SET transactional=false";
+ conn.createStatement().execute(ddl);
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.TX_MAY_NOT_SWITCH_TO_NON_TX.getErrorCode(), e.getErrorCode());
+ }
+
+ HBaseAdmin admin = pconn.getQueryServices().getAdmin();
+ HTableDescriptor desc = new HTableDescriptor(TableName.valueOf("TXN_TEST_EXISTING"));
+ desc.addFamily(new HColumnDescriptor(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES));
+ admin.createTable(desc);
+ ddl = "CREATE TABLE TXN_TEST_EXISTING (k varchar primary key) transactional=true";
+ conn.createStatement().execute(ddl);
+ assertEquals(Boolean.TRUE.toString(), admin.getTableDescriptor(TableName.valueOf("TXN_TEST_EXISTING")).getValue(TxConstants.READ_NON_TX_DATA));
+
+ // Should be ok, as HBase metadata should match existing metadata.
+ ddl = "CREATE TABLE IF NOT EXISTS TEST_TRANSACTIONAL_TABLE (k varchar primary key)";
+ try {
+ conn.createStatement().execute(ddl);
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.TX_MAY_NOT_SWITCH_TO_NON_TX.getErrorCode(), e.getErrorCode());
+ }
+ ddl += " transactional=true";
+ conn.createStatement().execute(ddl);
+ table = pconn.getTable(new PTableKey(null, "TEST_TRANSACTIONAL_TABLE"));
+ htable = pconn.getQueryServices().getTable(Bytes.toBytes("TEST_TRANSACTIONAL_TABLE"));
+ assertTrue(table.isTransactional());
+ assertTrue(htable.getTableDescriptor().getCoprocessors().contains(TransactionProcessor.class.getName()));
+ }
+
+ public void testCurrentDate() throws Exception {
+ String selectSql = "SELECT current_date() FROM "+FULL_TABLE_NAME;
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(false);
+ ResultSet rs = conn.createStatement().executeQuery(selectSql);
+ assertFalse(rs.next());
+
+ String upsert = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn.prepareStatement(upsert);
+ // upsert two rows
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.execute();
+ conn.commit();
+
+ rs = conn.createStatement().executeQuery(selectSql);
+ assertTrue(rs.next());
+ Date date1 = rs.getDate(1);
+ assertFalse(rs.next());
+
+ Thread.sleep(1000);
+
+ rs = conn.createStatement().executeQuery(selectSql);
+ assertTrue(rs.next());
+ Date date2 = rs.getDate(1);
+ assertFalse(rs.next());
+ assertTrue("current_date() should change while executing multiple statements", date2.getTime() > date1.getTime());
+ }
+ }
+
+ @Test
+ public void testReCreateTxnTableAfterDroppingExistingNonTxnTable() throws SQLException {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE DEMO(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
+ stmt.execute("DROP TABLE DEMO");
+ stmt.execute("CREATE TABLE DEMO(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) TRANSACTIONAL=true");
+ stmt.execute("CREATE INDEX DEMO_IDX ON DEMO (v1) INCLUDE(v2)");
+ assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "DEMO")).isTransactional());
+ assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "DEMO_IDX")).isTransactional());
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java
new file mode 100644
index 0000000..34869dd
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.tx;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.end2end.Shadower;
+import org.apache.phoenix.execute.MutationState;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import co.cask.tephra.Transaction.VisibilityLevel;
+
+import com.google.common.collect.Maps;
+
+@RunWith(Parameterized.class)
+public class TxCheckpointIT extends BaseHBaseManagedTimeIT {
+
+ private final boolean localIndex;
+ private final boolean mutable;
+ private String tableName;
+ private String indexName;
+ private String seqName;
+ private String fullTableName;
+
+ public TxCheckpointIT(boolean localIndex, boolean mutable) {
+ this.localIndex = localIndex;
+ this.mutable = mutable;
+ this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME;
+ this.indexName = "IDX_" + System.currentTimeMillis();
+ this.seqName = "SEQ_" + System.currentTimeMillis();
+ this.fullTableName = SchemaUtil.getTableName(tableName, tableName);
+ }
+
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(1);
+ props.put(QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @Parameters(name="localIndex = {0} , mutable = {1}")
+ public static Collection<Boolean[]> data() {
+ return Arrays.asList(new Boolean[][] {
+ { false, false }, { false, true }, { true, false }, { true, true }
+ });
+ }
+
+ @Test
+ public void testUpsertSelectDoesntSeeUpsertedData() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(QueryServices.MUTATE_BATCH_SIZE_ATTRIB, Integer.toString(3));
+ props.setProperty(QueryServices.SCAN_CACHE_SIZE_ATTRIB, Integer.toString(3));
+ props.setProperty(QueryServices.SCAN_RESULT_CHUNK_SIZE, Integer.toString(3));
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(true);
+ conn.createStatement().execute("CREATE SEQUENCE "+seqName);
+ conn.createStatement().execute("CREATE TABLE " + fullTableName + "(pk INTEGER PRIMARY KEY, val INTEGER)"+(!mutable? " IMMUTABLE_ROWS=true" : ""));
+ conn.createStatement().execute("CREATE "+(localIndex? "LOCAL " : "")+"INDEX " + indexName + " ON " + fullTableName + "(val)");
+
+ conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES (NEXT VALUE FOR " + seqName + ",1)");
+ for (int i=0; i<6; i++) {
+ Statement stmt = conn.createStatement();
+ int upsertCount = stmt.executeUpdate("UPSERT INTO " + fullTableName + " SELECT NEXT VALUE FOR " + seqName + ", val FROM " + fullTableName);
+ assertEquals((int)Math.pow(2, i), upsertCount);
+ }
+ conn.close();
+ }
+
+ @Test
+ public void testRollbackOfUncommittedDelete() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"+(!mutable? " IMMUTABLE_ROWS=true" : ""));
+ stmt.execute("CREATE "+(localIndex? "LOCAL " : "")+"INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE(v2)");
+
+ stmt.executeUpdate("upsert into " + fullTableName + " values('x1', 'y1', 'a1')");
+ stmt.executeUpdate("upsert into " + fullTableName + " values('x2', 'y2', 'a2')");
+
+ //assert values in data table
+ ResultSet rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName + " ORDER BY k");
+ assertTrue(rs.next());
+ assertEquals("x1", rs.getString(1));
+ assertEquals("y1", rs.getString(2));
+ assertEquals("a1", rs.getString(3));
+ assertTrue(rs.next());
+ assertEquals("x2", rs.getString(1));
+ assertEquals("y2", rs.getString(2));
+ assertEquals("a2", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert values in index table
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x1", rs.getString(1));
+ assertEquals("y1", rs.getString(2));
+ assertEquals("a1", rs.getString(3));
+ assertTrue(rs.next());
+ assertEquals("x2", rs.getString(1));
+ assertEquals("y2", rs.getString(2));
+ assertEquals("a2", rs.getString(3));
+ assertFalse(rs.next());
+
+ conn.commit();
+
+ stmt.executeUpdate("DELETE FROM " + fullTableName + " WHERE k='x1' AND v1='y1' AND v2='a1'");
+ //assert row is delete in data table
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName + " ORDER BY k");
+ assertTrue(rs.next());
+ assertEquals("x2", rs.getString(1));
+ assertEquals("y2", rs.getString(2));
+ assertEquals("a2", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert row is delete in index table
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x2", rs.getString(1));
+ assertEquals("y2", rs.getString(2));
+ assertEquals("a2", rs.getString(3));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ //assert two rows in data table
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName + " ORDER BY k");
+ assertTrue(rs.next());
+ assertEquals("x1", rs.getString(1));
+ assertEquals("y1", rs.getString(2));
+ assertEquals("a1", rs.getString(3));
+ assertTrue(rs.next());
+ assertEquals("x2", rs.getString(1));
+ assertEquals("y2", rs.getString(2));
+ assertEquals("a2", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert two rows in index table
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x1", rs.getString(1));
+ assertEquals("y1", rs.getString(2));
+ assertEquals("a1", rs.getString(3));
+ assertTrue(rs.next());
+ assertEquals("x2", rs.getString(1));
+ assertEquals("y2", rs.getString(2));
+ assertEquals("a2", rs.getString(3));
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testCheckpointForUpsertSelect() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);) {
+ conn.setAutoCommit(false);
+ Statement stmt = conn.createStatement();
+
+ stmt.execute("CREATE TABLE " + fullTableName + "(ID BIGINT NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"
+ + (!mutable ? " IMMUTABLE_ROWS=true" : ""));
+ stmt.execute("CREATE " + (localIndex ? "LOCAL " : "")
+ + "INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE(v2)");
+
+ stmt.executeUpdate("upsert into " + fullTableName + " values(1, 'a2', 'b1')");
+ stmt.executeUpdate("upsert into " + fullTableName + " values(2, 'a2', 'b2')");
+ stmt.executeUpdate("upsert into " + fullTableName + " values(3, 'a3', 'b3')");
+ conn.commit();
+
+ upsertRows(conn);
+ conn.rollback();
+ verifyRows(conn, 3);
+
+ upsertRows(conn);
+ conn.commit();
+ verifyRows(conn, 6);
+ }
+ }
+
+ private void verifyRows(Connection conn, int expectedMaxId) throws SQLException {
+ ResultSet rs;
+ //query the data table
+ rs = conn.createStatement().executeQuery("select /*+ NO_INDEX */ max(id) from " + fullTableName + "");
+ assertTrue(rs.next());
+ assertEquals(expectedMaxId, rs.getLong(1));
+ assertFalse(rs.next());
+
+ // query the index
+ rs = conn.createStatement().executeQuery("select /*+ INDEX(DEMO IDX) */ max(id) from " + fullTableName + "");
+ assertTrue(rs.next());
+ assertEquals(expectedMaxId, rs.getLong(1));
+ assertFalse(rs.next());
+ }
+
+ private void upsertRows(Connection conn) throws SQLException {
+ ResultSet rs;
+ MutationState state = conn.unwrap(PhoenixConnection.class)
+ .getMutationState();
+ state.startTransaction();
+ long wp = state.getWritePointer();
+ conn.createStatement().execute(
+ "upsert into " + fullTableName + " select max(id)+1, 'a4', 'b4' from " + fullTableName + "");
+ assertEquals(VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT,
+ state.getVisibilityLevel());
+ assertEquals(wp, state.getWritePointer()); // Make sure write ptr
+ // didn't move
+ rs = conn.createStatement().executeQuery("select max(id) from " + fullTableName + "");
+
+ assertTrue(rs.next());
+ assertEquals(4, rs.getLong(1));
+ assertFalse(rs.next());
+
+ conn.createStatement().execute(
+ "upsert into " + fullTableName + " select max(id)+1, 'a5', 'b5' from " + fullTableName + "");
+ assertEquals(VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT,
+ state.getVisibilityLevel());
+ assertNotEquals(wp, state.getWritePointer()); // Make sure write ptr
+ // moves
+ wp = state.getWritePointer();
+ rs = conn.createStatement().executeQuery("select max(id) from " + fullTableName + "");
+
+ assertTrue(rs.next());
+ assertEquals(5, rs.getLong(1));
+ assertFalse(rs.next());
+
+ conn.createStatement().execute(
+ "upsert into " + fullTableName + " select max(id)+1, 'a6', 'b6' from " + fullTableName + "");
+ assertEquals(VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT,
+ state.getVisibilityLevel());
+ assertNotEquals(wp, state.getWritePointer()); // Make sure write ptr
+ // moves
+ wp = state.getWritePointer();
+ rs = conn.createStatement().executeQuery("select max(id) from " + fullTableName + "");
+
+ assertTrue(rs.next());
+ assertEquals(6, rs.getLong(1));
+ assertFalse(rs.next());
+ }
+
+ @Test
+ public void testCheckpointForDeleteAndUpsert() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ ResultSet rs;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);) {
+ conn.setAutoCommit(false);
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName + "1(ID1 BIGINT NOT NULL PRIMARY KEY, FK1A INTEGER, FK1B INTEGER)"
+ + (!mutable ? " IMMUTABLE_ROWS=true" : ""));
+ stmt.execute("CREATE TABLE " + fullTableName + "2(ID2 BIGINT NOT NULL PRIMARY KEY, FK2 INTEGER)"
+ + (!mutable ? " IMMUTABLE_ROWS=true" : ""));
+ stmt.execute("CREATE " + (localIndex ? "LOCAL " : "")
+ + "INDEX " + indexName + " ON " + fullTableName + "1 (FK1B)");
+
+ stmt.executeUpdate("upsert into " + fullTableName + "1 values (1, 3, 3)");
+ stmt.executeUpdate("upsert into " + fullTableName + "1 values (2, 2, 2)");
+ stmt.executeUpdate("upsert into " + fullTableName + "1 values (3, 1, 1)");
+ stmt.executeUpdate("upsert into " + fullTableName + "2 values (1, 1)");
+ conn.commit();
+
+ MutationState state = conn.unwrap(PhoenixConnection.class).getMutationState();
+ state.startTransaction();
+ long wp = state.getWritePointer();
+ conn.createStatement().execute("delete from " + fullTableName + "1 where id1=fk1b AND fk1b=id1");
+ assertEquals(VisibilityLevel.SNAPSHOT, state.getVisibilityLevel());
+ assertEquals(wp, state.getWritePointer()); // Make sure write ptr didn't move
+
+ rs = conn.createStatement().executeQuery("select /*+ NO_INDEX */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(1,rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(3,rs.getLong(1));
+ assertFalse(rs.next());
+
+ rs = conn.createStatement().executeQuery("select /*+ INDEX(DEMO IDX) */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(3,rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(1,rs.getLong(1));
+ assertFalse(rs.next());
+
+ conn.createStatement().execute("delete from " + fullTableName + "1 where id1 in (select fk1a from " + fullTableName + "1 join " + fullTableName + "2 on (fk2=id1))");
+ assertEquals(VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT, state.getVisibilityLevel());
+ assertNotEquals(wp, state.getWritePointer()); // Make sure write ptr moved
+
+ rs = conn.createStatement().executeQuery("select /*+ NO_INDEX */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(1,rs.getLong(1));
+ assertFalse(rs.next());
+
+ rs = conn.createStatement().executeQuery("select /*+ INDEX(DEMO IDX) */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(1,rs.getLong(1));
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName + "1 SELECT id1 + 3, id1, id1 FROM " + fullTableName + "1");
+ stmt.executeUpdate("upsert into " + fullTableName + "2 values (2, 4)");
+
+ conn.createStatement().execute("delete from " + fullTableName + "1 where id1 in (select fk1a from " + fullTableName + "1 join " + fullTableName + "2 on (fk2=id1))");
+ assertEquals(VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT, state.getVisibilityLevel());
+ assertNotEquals(wp, state.getWritePointer()); // Make sure write ptr moved
+
+ rs = conn.createStatement().executeQuery("select /*+ NO_INDEX */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(4,rs.getLong(1));
+ assertFalse(rs.next());
+
+ rs = conn.createStatement().executeQuery("select /*+ INDEX(DEMO IDX) */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(4,rs.getLong(1));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ rs = conn.createStatement().executeQuery("select /*+ NO_INDEX */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(1,rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(2,rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(3,rs.getLong(1));
+ assertFalse(rs.next());
+
+ rs = conn.createStatement().executeQuery("select /*+ INDEX(DEMO IDX) */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(3,rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(2,rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(1,rs.getLong(1));
+ assertFalse(rs.next());
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/tx/TxPointInTimeQueryIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TxPointInTimeQueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/TxPointInTimeQueryIT.java
new file mode 100644
index 0000000..a3b3aa2
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TxPointInTimeQueryIT.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.tx;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import org.apache.phoenix.end2end.BaseClientManagedTimeIT;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TxPointInTimeQueryIT extends BaseClientManagedTimeIT {
+
+ protected long ts;
+
+ @Before
+ public void initTable() throws Exception {
+ ts = nextTimestamp();
+ }
+
+ @Test
+ public void testQueryWithSCN() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts));
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);) {
+ try {
+ conn.createStatement().execute(
+ "CREATE TABLE t (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR) TRANSACTIONAL=true");
+ fail();
+ } catch (SQLException e) {
+ assertEquals("Unexpected Exception",
+ SQLExceptionCode.CANNOT_START_TRANSACTION_WITH_SCN_SET
+ .getErrorCode(), e.getErrorCode());
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/cache/IndexMetaDataCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/cache/IndexMetaDataCache.java b/phoenix-core/src/main/java/org/apache/phoenix/cache/IndexMetaDataCache.java
index a1563db..359e08f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/IndexMetaDataCache.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/IndexMetaDataCache.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import java.util.Collections;
import java.util.List;
+import co.cask.tephra.Transaction;
+
import org.apache.phoenix.index.IndexMaintainer;
public interface IndexMetaDataCache extends Closeable {
@@ -36,6 +38,13 @@ public interface IndexMetaDataCache extends Closeable {
public List<IndexMaintainer> getIndexMaintainers() {
return Collections.emptyList();
}
+
+ @Override
+ public Transaction getTransaction() {
+ return null;
+ }
+
};
public List<IndexMaintainer> getIndexMaintainers();
+ public Transaction getTransaction();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java b/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
index 3cfdd71..f188ab2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
@@ -71,6 +71,7 @@ import org.apache.phoenix.util.SQLCloseables;
import org.apache.phoenix.util.ScanUtil;
import com.google.common.collect.ImmutableSet;
+import com.google.protobuf.HBaseZeroCopyByteString;
/**
*
@@ -144,7 +145,7 @@ public class ServerCacheClient {
}
- public ServerCache addServerCache(ScanRanges keyRanges, final ImmutableBytesWritable cachePtr, final ServerCacheFactory cacheFactory, final TableRef cacheUsingTableRef) throws SQLException {
+ public ServerCache addServerCache(ScanRanges keyRanges, final ImmutableBytesWritable cachePtr, final byte[] txState, final ServerCacheFactory cacheFactory, final TableRef cacheUsingTableRef) throws SQLException {
ConnectionQueryServices services = connection.getQueryServices();
MemoryChunk chunk = services.getMemoryManager().allocate(cachePtr.getLength());
List<Closeable> closeables = new ArrayList<Closeable>();
@@ -213,6 +214,7 @@ public class ServerCacheClient {
ServerCacheFactoryProtos.ServerCacheFactory.Builder svrCacheFactoryBuider = ServerCacheFactoryProtos.ServerCacheFactory.newBuilder();
svrCacheFactoryBuider.setClassName(cacheFactory.getClass().getName());
builder.setCacheFactory(svrCacheFactoryBuider.build());
+ builder.setTxState(HBaseZeroCopyByteString.wrap(txState));
instance.addServerCache(controller, builder.build(), rpcCallback);
if(controller.getFailedOn() != null) {
throw controller.getFailedOn();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCache.java b/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCache.java
index b968a9b..c7cd58f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCache.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCache.java
@@ -37,6 +37,6 @@ import org.apache.phoenix.memory.MemoryManager;
public interface TenantCache {
MemoryManager getMemoryManager();
Closeable getServerCache(ImmutableBytesPtr cacheId);
- Closeable addServerCache(ImmutableBytesPtr cacheId, ImmutableBytesWritable cachePtr, ServerCacheFactory cacheFactory) throws SQLException;
+ Closeable addServerCache(ImmutableBytesPtr cacheId, ImmutableBytesWritable cachePtr, byte[] txState, ServerCacheFactory cacheFactory) throws SQLException;
void removeServerCache(ImmutableBytesPtr cacheId) throws SQLException;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
index 9005fa8..6ef7a6f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
@@ -23,14 +23,17 @@ import java.util.Collections;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
-import com.google.common.cache.*;
import org.apache.phoenix.coprocessor.ServerCachingProtocol.ServerCacheFactory;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.memory.MemoryManager;
import org.apache.phoenix.memory.MemoryManager.MemoryChunk;
import org.apache.phoenix.util.Closeables;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+
/**
*
* Cache per tenant on server side. Tracks memory usage for each
@@ -80,11 +83,11 @@ public class TenantCacheImpl implements TenantCache {
}
@Override
- public Closeable addServerCache(ImmutableBytesPtr cacheId, ImmutableBytesWritable cachePtr, ServerCacheFactory cacheFactory) throws SQLException {
- MemoryChunk chunk = this.getMemoryManager().allocate(cachePtr.getLength());
+ public Closeable addServerCache(ImmutableBytesPtr cacheId, ImmutableBytesWritable cachePtr, byte[] txState, ServerCacheFactory cacheFactory) throws SQLException {
+ MemoryChunk chunk = this.getMemoryManager().allocate(cachePtr.getLength() + txState.length);
boolean success = false;
try {
- Closeable element = cacheFactory.newCache(cachePtr, chunk);
+ Closeable element = cacheFactory.newCache(cachePtr, txState, chunk);
getServerCaches().put(cacheId, element);
success = true;
return element;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
index f09b508..d5b74fb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
@@ -132,8 +132,8 @@ public class CreateTableCompiler {
// on our connection.
new DelegateConnectionQueryServices(connection.getQueryServices()) {
@Override
- public PMetaData addTable(PTable table) throws SQLException {
- return connection.addTable(table);
+ public PMetaData addTable(PTable table, long resolvedTime) throws SQLException {
+ return connection.addTable(table, resolvedTime);
}
},
connection, tableRef.getTimeStamp());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
index f0f693e..12aa7c5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
@@ -78,6 +78,7 @@ import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PLong;
+import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.ScanUtil;
@@ -323,16 +324,20 @@ public class DeleteCompiler {
// - read-only VIEW
// - transactional table with a connection having an SCN
// TODO: SchemaUtil.isReadOnly(PTable, connection)?
- if ( table.getType() == PTableType.VIEW && table.getViewType().isReadOnly() ) {
+ if (table.getType() == PTableType.VIEW && table.getViewType().isReadOnly()) {
throw new ReadOnlyTableException(schemaName,tableName);
}
+ else if (table.isTransactional() && connection.getSCN() != null) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_SPECIFY_SCN_FOR_TXN_TABLE).setSchemaName(schemaName)
+ .setTableName(tableName).build().buildException();
+ }
immutableIndex = getNonDisabledImmutableIndexes(tableRefToBe);
boolean mayHaveImmutableIndexes = !immutableIndex.isEmpty();
noQueryReqd = !hasLimit;
// Can't run on same server for transactional data, as we need the row keys for the data
// that is being upserted for conflict detection purposes.
- runOnServer = isAutoCommit && noQueryReqd;
+ runOnServer = isAutoCommit && noQueryReqd && !table.isTransactional();
HintNode hint = delete.getHint();
if (runOnServer && !delete.getHint().hasHint(Hint.USE_INDEX_OVER_DATA_TABLE)) {
hint = HintNode.create(hint, Hint.USE_DATA_OVER_INDEX_TABLE);
@@ -363,8 +368,8 @@ public class DeleteCompiler {
QueryCompiler compiler = new QueryCompiler(statement, select, resolverToBe, Collections.<PColumn>emptyList(), parallelIteratorFactory, new SequenceManager(statement));
dataPlanToBe = compiler.compile();
queryPlans = Lists.newArrayList(mayHaveImmutableIndexes
- ? optimizer.getApplicablePlans(statement, select, resolverToBe, Collections.<PColumn>emptyList(), parallelIteratorFactory)
- : optimizer.getBestPlan(statement, select, resolverToBe, Collections.<PColumn>emptyList(), parallelIteratorFactory));
+ ? optimizer.getApplicablePlans(dataPlanToBe, statement, select, resolverToBe, Collections.<PColumn>emptyList(), parallelIteratorFactory)
+ : optimizer.getBestPlan(dataPlanToBe, statement, select, resolverToBe, Collections.<PColumn>emptyList(), parallelIteratorFactory));
if (mayHaveImmutableIndexes) { // FIXME: this is ugly
// Lookup the table being deleted from in the cache, as it's possible that the
// optimizer updated the cache if it found indexes that were out of date.
@@ -538,11 +543,12 @@ public class DeleteCompiler {
ImmutableBytesWritable ptr = context.getTempPtr();
PTable table = tableRef.getTable();
table.getIndexMaintainers(ptr, context.getConnection());
+ byte[] txState = table.isTransactional() ? connection.getMutationState().encodeTransaction() : ByteUtil.EMPTY_BYTE_ARRAY;
ServerCache cache = null;
try {
if (ptr.getLength() > 0) {
IndexMetaDataCacheClient client = new IndexMetaDataCacheClient(connection, tableRef);
- cache = client.addIndexMetadataCache(context.getScanRanges(), ptr);
+ cache = client.addIndexMetadataCache(context.getScanRanges(), ptr, txState);
byte[] uuidValue = cache.getId();
context.getScan().setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
index 426556f..0828b94 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
@@ -80,6 +80,7 @@ import org.apache.phoenix.util.Closeables;
import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.LogUtil;
import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TransactionUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -226,7 +227,7 @@ public class FromCompiler {
PTable t = PTableImpl.makePTable(table, projectedColumns);
return new SingleTableColumnResolver(connection, new TableRef(tableRef.getTableAlias(), t, tableRef.getLowerBoundTimeStamp(), tableRef.hasDynamicCols()));
}
-
+
public static ColumnResolver getResolver(TableRef tableRef)
throws SQLException {
SingleTableColumnResolver visitor = new SingleTableColumnResolver(tableRef);
@@ -407,7 +408,7 @@ public class FromCompiler {
PTable theTable = null;
if (updateCacheImmediately || connection.getAutoCommit()) {
MetaDataMutationResult result = client.updateCache(schemaName, tableName);
- timeStamp = result.getMutationTime();
+ timeStamp = TransactionUtil.getResolvedTimestamp(connection, result);
theTable = result.getTable();
if (theTable == null) {
throw new TableNotFoundException(schemaName, tableName, timeStamp);
@@ -427,7 +428,7 @@ public class FromCompiler {
if (theTable == null) {
MetaDataMutationResult result = client.updateCache(schemaName, tableName);
if (result.wasUpdated()) {
- timeStamp = result.getMutationTime();
+ timeStamp = TransactionUtil.getResolvedTimestamp(connection, result);
theTable = result.getTable();
}
}
@@ -651,7 +652,7 @@ public class FromCompiler {
PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM,
null, null, columns, null, null, Collections.<PTable>emptyList(),
false, Collections.<PName>emptyList(), null, null, false, false, false, null,
- null, null, false);
+ null, null, false, false);
String alias = subselectNode.getAlias();
TableRef tableRef = new TableRef(alias, t, MetaDataProtocol.MIN_TABLE_TIMESTAMP, false);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
index e4ca5d9..b55e4aa 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
@@ -1302,7 +1302,7 @@ public class JoinCompiler {
left.getBucketNum(), merged,left.getParentSchemaName(), left.getParentTableName(), left.getIndexes(),
left.isImmutableRows(), Collections.<PName>emptyList(), null, null, PTable.DEFAULT_DISABLE_WAL,
left.isMultiTenant(), left.getStoreNulls(), left.getViewType(), left.getViewIndexId(), left.getIndexType(),
- left.rowKeyOrderOptimizable());
+ left.rowKeyOrderOptimizable(), left.isTransactional());
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
index c6f6bf2..506623b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
@@ -48,6 +48,7 @@ import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.util.ScanUtil;
+import org.apache.phoenix.util.TransactionUtil;
import com.google.common.collect.Lists;
@@ -179,7 +180,14 @@ public class PostDDLCompiler {
};
PhoenixStatement statement = new PhoenixStatement(connection);
StatementContext context = new StatementContext(statement, resolver, scan, new SequenceManager(statement));
- ScanUtil.setTimeRange(scan, timestamp);
+ long ts = timestamp;
+ // FIXME: DDL operations aren't transactional, so we're basing the timestamp on a server timestamp.
+ // Not sure what the fix should be. We don't need conflict detection nor filtering of invalid transactions
+ // in this case, so maybe this is ok.
+ if (tableRef.getTable().isTransactional()) {
+ ts = TransactionUtil.convertToNanoseconds(ts);
+ }
+ ScanUtil.setTimeRange(scan, ts);
if (emptyCF != null) {
scan.setAttribute(BaseScannerRegionObserver.EMPTY_CF, emptyCF);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
index 312de45..80c4b89 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
@@ -252,7 +252,8 @@ public class StatementContext {
public long getCurrentTime() throws SQLException {
long ts = this.getCurrentTable().getTimeStamp();
- if (ts != QueryConstants.UNSET_TIMESTAMP) {
+ // if the table is transactional then it is only resolved once per query, so we can't use the table timestamp
+ if (!this.getCurrentTable().getTable().isTransactional() && ts != QueryConstants.UNSET_TIMESTAMP) {
return ts;
}
if (currentTime != QueryConstants.UNSET_TIMESTAMP) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
index c6aa546..551b05c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
@@ -152,7 +152,7 @@ public class TupleProjectionCompiler {
table.getBucketNum(), projectedColumns, table.getParentSchemaName(),
table.getParentName(), table.getIndexes(), table.isImmutableRows(), Collections.<PName>emptyList(), null, null,
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(),
- table.getIndexType(), table.rowKeyOrderOptimizable());
+ table.getIndexType(), table.rowKeyOrderOptimizable(), table.isTransactional());
}
public static PTable createProjectedTable(TableRef tableRef, List<ColumnRef> sourceColumnRefs, boolean retainPKColumns) throws SQLException {
@@ -179,7 +179,7 @@ public class TupleProjectionCompiler {
retainPKColumns ? table.getBucketNum() : null, projectedColumns, null,
null, Collections.<PTable>emptyList(), table.isImmutableRows(), Collections.<PName>emptyList(), null, null,
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(),
- null, table.rowKeyOrderOptimizable());
+ null, table.rowKeyOrderOptimizable(), table.isTransactional());
}
// For extracting column references from single select statement
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
index afca97a..298303d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
@@ -82,7 +82,7 @@ public class UnionCompiler {
PTable tempTable = PTableImpl.makePTable(statement.getConnection().getTenantId(), UNION_SCHEMA_NAME, UNION_TABLE_NAME,
PTableType.SUBQUERY, null, HConstants.LATEST_TIMESTAMP, scn == null ? HConstants.LATEST_TIMESTAMP : scn, null, null,
projectedColumns, null, null, null,
- true, null, null, null, true, true, true, null, null, null, false);
+ true, null, null, null, true, true, true, null, null, null, false, false);
TableRef tableRef = new TableRef(null, tempTable, 0, false);
return tableRef;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
index 953eb2f..0f7f6f9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
@@ -182,7 +182,7 @@ public class UpsertCompiler {
if (isAutoCommit && rowCount % batchSize == 0) {
MutationState state = new MutationState(tableRef, mutation, 0, maxSize, connection);
connection.getMutationState().join(state);
- connection.commit();
+ connection.getMutationState().send();
mutation.clear();
}
}
@@ -292,9 +292,13 @@ public class UpsertCompiler {
// Cannot update:
// - read-only VIEW
// - transactional table with a connection having an SCN
- if ( table.getType() == PTableType.VIEW && table.getViewType().isReadOnly() ) {
+ if (table.getType() == PTableType.VIEW && table.getViewType().isReadOnly()) {
throw new ReadOnlyTableException(schemaName,tableName);
}
+ else if (table.isTransactional() && connection.getSCN() != null) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_SPECIFY_SCN_FOR_TXN_TABLE).setSchemaName(schemaName)
+ .setTableName(tableName).build().buildException();
+ }
boolean isSalted = table.getBucketNum() != null;
isTenantSpecific = table.isMultiTenant() && connection.getTenantId() != null;
isSharedViewIndex = table.getViewIndexId() != null;
@@ -463,7 +467,7 @@ public class UpsertCompiler {
// so we might be able to run it entirely on the server side.
// For a table with row timestamp column, we can't guarantee that the row key will reside in the
// region space managed by region servers. So we bail out on executing on server side.
- runOnServer = sameTable && isAutoCommit && !(table.isImmutableRows() && !table.getIndexes().isEmpty()) && table.getRowTimestampColPos() == -1;
+ runOnServer = sameTable && isAutoCommit && !table.isTransactional() && !(table.isImmutableRows() && !table.getIndexes().isEmpty()) && table.getRowTimestampColPos() == -1;
}
// If we may be able to run on the server, add a hint that favors using the data table
// if all else is equal.
@@ -678,12 +682,13 @@ public class UpsertCompiler {
ImmutableBytesWritable ptr = context.getTempPtr();
PTable table = tableRef.getTable();
table.getIndexMaintainers(ptr, context.getConnection());
+ byte[] txState = table.isTransactional() ? connection.getMutationState().encodeTransaction() : ByteUtil.EMPTY_BYTE_ARRAY;
ServerCache cache = null;
try {
if (ptr.getLength() > 0) {
IndexMetaDataCacheClient client = new IndexMetaDataCacheClient(connection, tableRef);
- cache = client.addIndexMetadataCache(context.getScanRanges(), ptr);
+ cache = client.addIndexMetadataCache(context.getScanRanges(), ptr, txState);
byte[] uuidValue = cache.getId();
scan.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index 33218ee..d720806 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -57,6 +57,8 @@ import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.ServerUtil;
+import co.cask.tephra.Transaction;
+
import com.google.common.collect.ImmutableList;
@@ -86,6 +88,7 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
public static final String EXPECTED_UPPER_REGION_KEY = "_ExpectedUpperRegionKey";
public static final String REVERSE_SCAN = "_ReverseScan";
public static final String ANALYZE_TABLE = "_ANALYZETABLE";
+ public static final String TX_STATE = "_TxState";
public static final String GUIDEPOST_WIDTH_BYTES = "_GUIDEPOST_WIDTH_BYTES";
public static final String GUIDEPOST_PER_REGION = "_GUIDEPOST_PER_REGION";
public static final String UPGRADE_DESC_ROW_KEY = "_UPGRADE_DESC_ROW_KEY";
@@ -228,7 +231,7 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
final byte[][] viewConstants, final TupleProjector projector,
final ImmutableBytesWritable ptr) {
return getWrappedScanner(c, s, null, null, offset, scan, dataColumns, tupleProjector,
- dataRegion, indexMaintainer, viewConstants, null, null, projector, ptr);
+ dataRegion, indexMaintainer, null, viewConstants, null, null, projector, ptr);
}
/**
@@ -236,13 +239,14 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
* re-throws as DoNotRetryIOException to prevent needless retrying hanging the query
* for 30 seconds. Unfortunately, until HBASE-7481 gets fixed, there's no way to do
* the same from a custom filter.
- * @param arrayFuncRefs
* @param arrayKVRefs
+ * @param arrayFuncRefs
* @param offset starting position in the rowkey.
* @param scan
* @param tupleProjector
* @param dataRegion
* @param indexMaintainer
+ * @param tx current transaction
* @param viewConstants
*/
protected RegionScanner getWrappedScanner(final ObserverContext<RegionCoprocessorEnvironment> c,
@@ -250,6 +254,7 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
final Expression[] arrayFuncRefs, final int offset, final Scan scan,
final ColumnReference[] dataColumns, final TupleProjector tupleProjector,
final Region dataRegion, final IndexMaintainer indexMaintainer,
+ Transaction tx,
final byte[][] viewConstants, final KeyValueSchema kvSchema,
final ValueBitSet kvSchemaBitSet, final TupleProjector projector,
final ImmutableBytesWritable ptr) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 7eb1dc6..0a313be 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -60,6 +60,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME_INDEX;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID_INDEX;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_BYTES;
@@ -240,6 +241,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
private static final KeyValue EMPTY_KEYVALUE_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES);
private static final KeyValue BASE_COLUMN_COUNT_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.BASE_COLUMN_COUNT_BYTES);
private static final KeyValue ROW_KEY_ORDER_OPTIMIZABLE_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, ROW_KEY_ORDER_OPTIMIZABLE_BYTES);
+ private static final KeyValue TRANSACTIONAL_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, TRANSACTIONAL_BYTES);
private static final List<KeyValue> TABLE_KV_COLUMNS = Arrays.<KeyValue>asList(
EMPTY_KEYVALUE_KV,
@@ -261,7 +263,8 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
INDEX_DISABLE_TIMESTAMP_KV,
STORE_NULLS_KV,
BASE_COLUMN_COUNT_KV,
- ROW_KEY_ORDER_OPTIMIZABLE_KV
+ ROW_KEY_ORDER_OPTIMIZABLE_KV,
+ TRANSACTIONAL_KV
);
static {
Collections.sort(TABLE_KV_COLUMNS, KeyValue.COMPARATOR);
@@ -285,6 +288,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
private static final int STORE_NULLS_INDEX = TABLE_KV_COLUMNS.indexOf(STORE_NULLS_KV);
private static final int BASE_COLUMN_COUNT_INDEX = TABLE_KV_COLUMNS.indexOf(BASE_COLUMN_COUNT_KV);
private static final int ROW_KEY_ORDER_OPTIMIZABLE_INDEX = TABLE_KV_COLUMNS.indexOf(ROW_KEY_ORDER_OPTIMIZABLE_KV);
+ private static final int TRANSACTIONAL_INDEX = TABLE_KV_COLUMNS.indexOf(TRANSACTIONAL_KV);
// KeyValues for Column
private static final KeyValue DECIMAL_DIGITS_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, DECIMAL_DIGITS_BYTES);
@@ -789,6 +793,8 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
boolean multiTenant = multiTenantKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(multiTenantKv.getValueArray(), multiTenantKv.getValueOffset(), multiTenantKv.getValueLength()));
Cell storeNullsKv = tableKeyValues[STORE_NULLS_INDEX];
boolean storeNulls = storeNullsKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(storeNullsKv.getValueArray(), storeNullsKv.getValueOffset(), storeNullsKv.getValueLength()));
+ Cell transactionalKv = tableKeyValues[TRANSACTIONAL_INDEX];
+ boolean transactional = transactionalKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(transactionalKv.getValueArray(), transactionalKv.getValueOffset(), transactionalKv.getValueLength()));
Cell viewTypeKv = tableKeyValues[VIEW_TYPE_INDEX];
ViewType viewType = viewTypeKv == null ? null : ViewType.fromSerializedValue(viewTypeKv.getValueArray()[viewTypeKv.getValueOffset()]);
Cell viewIndexIdKv = tableKeyValues[VIEW_INDEX_ID_INDEX];
@@ -844,7 +850,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
return PTableImpl.makePTable(tenantId, schemaName, tableName, tableType, indexState, timeStamp,
tableSeqNum, pkName, saltBucketNum, columns, tableType == INDEX ? schemaName : null,
tableType == INDEX ? dataTableName : null, indexes, isImmutableRows, physicalTables, defaultFamilyName, viewStatement,
- disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, stats, baseColumnCount, rowKeyOrderOptimizable);
+ disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, stats, baseColumnCount, rowKeyOrderOptimizable, transactional);
}
private PFunction getFunction(RegionScanner scanner, final boolean isReplace, long clientTimeStamp, List<Mutation> deleteMutationsForReplace)
@@ -2332,6 +2338,17 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
}
continue;
}
+ } else if (pkCount == COLUMN_NAME_INDEX &&
+ ! (Bytes.compareTo(schemaName, rowKeyMetaData[SCHEMA_NAME_INDEX]) == 0 &&
+ Bytes.compareTo(tableName, rowKeyMetaData[TABLE_NAME_INDEX]) == 0 ) ) {
+ // Invalidate any table with mutations
+ // TODO: this likely means we don't need the above logic that
+ // loops through the indexes if adding a PK column, since we'd
+ // always have header rows for those.
+ invalidateList.add(new ImmutableBytesPtr(SchemaUtil
+ .getTableKey(tenantId,
+ rowKeyMetaData[SCHEMA_NAME_INDEX],
+ rowKeyMetaData[TABLE_NAME_INDEX])));
}
}
tableMetaData.addAll(mutationsForAddingColumnsToViews);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index 7cc4123..ba06828 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -23,9 +23,9 @@ import java.util.List;
import org.apache.hadoop.hbase.util.ByteStringer;
import org.apache.phoenix.coprocessor.generated.MetaDataProtos;
-import org.apache.phoenix.coprocessor.generated.PFunctionProtos;
import org.apache.phoenix.coprocessor.generated.MetaDataProtos.MetaDataResponse;
import org.apache.phoenix.coprocessor.generated.MetaDataProtos.MetaDataService;
+import org.apache.phoenix.coprocessor.generated.PFunctionProtos;
import org.apache.phoenix.hbase.index.util.VersionUtil;
import org.apache.phoenix.parse.PFunction;
import org.apache.phoenix.schema.PColumn;
@@ -73,7 +73,9 @@ public abstract class MetaDataProtocol extends MetaDataService {
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0 = MIN_TABLE_TIMESTAMP + 7;
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0 = MIN_TABLE_TIMESTAMP + 8;
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0 = MIN_TABLE_TIMESTAMP + 9;
- public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0;
+ public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 = MIN_TABLE_TIMESTAMP + 10;
+ // MIN_SYSTEM_TABLE_TIMESTAMP needs to be set to the max of all the MIN_SYSTEM_TABLE_TIMESTAMP_* constants
+ public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0;
// TODO: pare this down to minimum, as we don't need duplicates for both table and column errors, nor should we need
// a different code for every type of error.
// ENTITY_ALREADY_EXISTS, ENTITY_NOT_FOUND, NEWER_ENTITY_FOUND, ENTITY_NOT_IN_REGION, CONCURRENT_MODIFICATION
[9/9] phoenix git commit: PHOENIX-1674 Snapshot isolation transaction
support through Tephra (James Taylor, Thomas D'Silva)
Posted by td...@apache.org.
PHOENIX-1674 Snapshot isolation transaction support through Tephra (James Taylor, Thomas D'Silva)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/cc9929b5
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/cc9929b5
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/cc9929b5
Branch: refs/heads/master
Commit: cc9929b5c54eb979294df70929aa6eb781748b46
Parents: 76a8d18
Author: Thomas D'Silva <td...@salesforce.com>
Authored: Mon Nov 23 21:24:01 2015 -0800
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Mon Nov 23 21:24:01 2015 -0800
----------------------------------------------------------------------
bin/tephra | 310 ++++++
bin/tephra-env.sh | 55 ++
phoenix-assembly/src/build/client.xml | 4 +
phoenix-assembly/src/build/server-with-txns.xml | 48 +
.../src/build/server-without-antlr.xml | 46 +
phoenix-assembly/src/build/server.xml | 10 +-
phoenix-core/pom.xml | 24 +
.../ConnectionQueryServicesTestImpl.java | 4 +-
.../StatsCollectorWithSplitsAndMultiCFIT.java | 10 +-
.../phoenix/end2end/index/ImmutableIndexIT.java | 24 +-
.../end2end/index/IndexExpressionIT.java | 5 +-
.../apache/phoenix/end2end/index/IndexIT.java | 112 +--
.../end2end/index/MutableIndexFailureIT.java | 490 ++++++----
.../phoenix/end2end/index/MutableIndexIT.java | 68 +-
.../end2end/index/txn/MutableRollbackIT.java | 510 ++++++++++
.../phoenix/end2end/index/txn/RollbackIT.java | 167 ++++
.../end2end/index/txn/TxWriteFailureIT.java | 214 +++++
.../apache/phoenix/execute/PartialCommitIT.java | 149 +--
.../org/apache/phoenix/rpc/UpdateCacheIT.java | 108 +--
.../phoenix/rpc/UpdateCacheWithScnIT.java | 41 +
.../org/apache/phoenix/tx/TransactionIT.java | 537 +++++++++++
.../org/apache/phoenix/tx/TxCheckpointIT.java | 378 ++++++++
.../apache/phoenix/tx/TxPointInTimeQueryIT.java | 62 ++
.../phoenix/cache/IndexMetaDataCache.java | 9 +
.../apache/phoenix/cache/ServerCacheClient.java | 4 +-
.../org/apache/phoenix/cache/TenantCache.java | 2 +-
.../apache/phoenix/cache/TenantCacheImpl.java | 13 +-
.../phoenix/compile/CreateTableCompiler.java | 4 +-
.../apache/phoenix/compile/DeleteCompiler.java | 16 +-
.../apache/phoenix/compile/FromCompiler.java | 9 +-
.../apache/phoenix/compile/JoinCompiler.java | 2 +-
.../apache/phoenix/compile/PostDDLCompiler.java | 10 +-
.../phoenix/compile/StatementContext.java | 3 +-
.../compile/TupleProjectionCompiler.java | 4 +-
.../apache/phoenix/compile/UnionCompiler.java | 2 +-
.../apache/phoenix/compile/UpsertCompiler.java | 13 +-
.../coprocessor/BaseScannerRegionObserver.java | 9 +-
.../coprocessor/MetaDataEndpointImpl.java | 21 +-
.../phoenix/coprocessor/MetaDataProtocol.java | 6 +-
.../phoenix/coprocessor/ScanRegionObserver.java | 12 +-
.../coprocessor/ServerCachingEndpointImpl.java | 8 +-
.../coprocessor/ServerCachingProtocol.java | 5 +-
.../UngroupedAggregateRegionObserver.java | 6 +
.../coprocessor/generated/PTableProtos.java | 103 +-
.../generated/ServerCachingProtos.java | 120 ++-
.../phoenix/exception/SQLExceptionCode.java | 17 +
.../apache/phoenix/execute/BaseQueryPlan.java | 40 +-
.../apache/phoenix/execute/DelegateHTable.java | 300 ++++++
.../apache/phoenix/execute/HashJoinPlan.java | 27 +-
.../apache/phoenix/execute/MutationState.java | 932 ++++++++++++++-----
.../org/apache/phoenix/execute/ScanPlan.java | 2 +-
.../apache/phoenix/index/IndexMaintainer.java | 4 +-
.../phoenix/index/IndexMetaDataCacheClient.java | 12 +-
.../index/IndexMetaDataCacheFactory.java | 19 +-
.../index/PhoenixIndexFailurePolicy.java | 2 +-
.../phoenix/index/PhoenixIndexMetaData.java | 13 +
.../index/PhoenixTransactionalIndexer.java | 505 ++++++++++
.../phoenix/iterate/BaseResultIterators.java | 5 +
.../phoenix/iterate/ChunkedResultIterator.java | 25 +-
.../phoenix/iterate/ParallelIterators.java | 2 +-
.../apache/phoenix/iterate/SerialIterators.java | 2 +-
.../phoenix/iterate/TableResultIterator.java | 28 +-
.../apache/phoenix/jdbc/PhoenixConnection.java | 77 +-
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 3 +
.../org/apache/phoenix/jdbc/PhoenixDriver.java | 2 +-
.../phoenix/jdbc/PhoenixEmbeddedDriver.java | 4 +
.../apache/phoenix/jdbc/PhoenixStatement.java | 15 +-
.../apache/phoenix/join/HashCacheClient.java | 3 +-
.../apache/phoenix/join/HashCacheFactory.java | 2 +-
.../phoenix/mapreduce/PhoenixRecordReader.java | 2 +-
.../index/PhoenixIndexImportDirectMapper.java | 2 +-
.../apache/phoenix/optimize/QueryOptimizer.java | 15 +-
.../phoenix/query/ConnectionQueryServices.java | 6 +-
.../query/ConnectionQueryServicesImpl.java | 621 +++++++++---
.../query/ConnectionlessQueryServicesImpl.java | 58 +-
.../query/DelegateConnectionQueryServices.java | 27 +-
.../apache/phoenix/query/MetaDataMutated.java | 7 +-
.../apache/phoenix/query/QueryConstants.java | 23 +-
.../org/apache/phoenix/query/QueryServices.java | 3 +
.../phoenix/query/QueryServicesOptions.java | 5 +-
.../apache/phoenix/schema/DelegateTable.java | 8 +-
.../apache/phoenix/schema/MetaDataClient.java | 376 +++++---
.../org/apache/phoenix/schema/PMetaData.java | 4 +-
.../apache/phoenix/schema/PMetaDataImpl.java | 107 +--
.../java/org/apache/phoenix/schema/PTable.java | 3 +-
.../org/apache/phoenix/schema/PTableImpl.java | 73 +-
.../org/apache/phoenix/schema/PTableRef.java | 65 ++
.../apache/phoenix/schema/TableProperty.java | 5 +-
.../org/apache/phoenix/schema/TableRef.java | 12 +-
.../phoenix/trace/PhoenixMetricsSink.java | 9 +-
.../java/org/apache/phoenix/util/IndexUtil.java | 29 +-
.../org/apache/phoenix/util/MetaDataUtil.java | 7 +
.../java/org/apache/phoenix/util/ScanUtil.java | 2 +-
.../org/apache/phoenix/util/SchemaUtil.java | 29 +-
.../org/apache/phoenix/util/ServerUtil.java | 5 +-
.../apache/phoenix/util/TransactionUtil.java | 105 +++
.../phoenix/compile/QueryCompilerTest.java | 2 +-
.../phoenix/compile/ViewCompilerTest.java | 29 +-
.../phoenix/execute/CorrelatePlanTest.java | 2 +-
.../phoenix/filter/SkipScanBigFilterTest.java | 3 +-
.../covered/CoveredIndexCodecForTesting.java | 67 +-
.../phoenix/index/IndexMaintainerTest.java | 2 +-
.../apache/phoenix/jdbc/PhoenixTestDriver.java | 4 +-
.../java/org/apache/phoenix/query/BaseTest.java | 130 ++-
.../phoenix/query/ConnectionlessTest.java | 5 +-
.../phoenix/schema/PMetaDataImplTest.java | 4 +-
.../java/org/apache/phoenix/util/TestUtil.java | 38 +-
phoenix-flume/pom.xml | 7 +
phoenix-pig/pom.xml | 7 +
phoenix-protocol/src/main/PTable.proto | 1 +
.../src/main/ServerCachingService.proto | 1 +
phoenix-server/pom.xml | 7 +
pom.xml | 5 +-
113 files changed, 6349 insertions(+), 1371 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/bin/tephra
----------------------------------------------------------------------
diff --git a/bin/tephra b/bin/tephra
new file mode 100644
index 0000000..54fe301
--- /dev/null
+++ b/bin/tephra
@@ -0,0 +1,310 @@
+#!/bin/sh
+
+#
+# Copyright © 2014 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+
+# Source function library. used for "status" use case
+if [ -f "/etc/rc.d/init.d/functions" ]; then
+ PLATFORM="RHEL"
+ . /etc/rc.d/init.d/functions
+elif [ -f /lib/lsb/init-functions ] ; then
+ PLATFORM="UBUNTU"
+ . /lib/lsb/init-functions
+else
+ PLATFORM="UNSUPPORTED"
+fi
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+bin=`dirname "${BASH_SOURCE-$0}"`
+bin=`cd "$bin"; pwd`
+lib="$bin"/../lib
+conf="$bin"/../conf
+script=`basename $0`
+
+# Resolve relative symlinks
+while [ -h "$PRG" ] ; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "$PRG"`"/$link"
+ fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/.."
+APP_HOME="`pwd -P`"
+APP=`basename $0`
+
+# Load component common environment file too
+. $bin/tephra-env.sh
+
+pid=$PID_DIR/tephra-service-${IDENT_STRING}.pid
+
+# In other environment, the jars are expected to be in <HOME>/lib directory.
+# Load all the jar files. Not ideal, but we need to load only the things that
+# is needed by this script.
+if [ "$CLASSPATH" = "" ]; then
+ CLASSPATH=${lib}/*
+else
+ CLASSPATH=$CLASSPATH:${lib}/*
+fi
+
+# Load the configuration too.
+if [ -d "$conf" ]; then
+ CLASSPATH=$CLASSPATH:"$conf"/
+fi
+
+# Set Log location
+if [ ! -e $LOG_DIR ]; then
+ mkdir -p $LOG_DIR;
+fi
+export LOG_PREFIX="tephra-service-$IDENT_STRING-$HOSTNAME"
+export LOGFILE=$LOG_PREFIX.log
+loglog="${LOG_DIR}/${LOGFILE}"
+
+# set the classpath to include hadoop and hbase dependencies
+set_classpath()
+{
+ COMP_HOME=$1
+ if [ -n "$HBASE_HOME" ]; then
+ HBASE_CP=`$HBASE_HOME/bin/hbase classpath`
+ elif [ `which hbase` ]; then
+ HBASE_CP=`hbase classpath`
+ fi
+
+ export HBASE_CP
+
+ if [ -n "$HBASE_CP" ]; then
+ CP=$COMP_HOME/phoenix-assembly/target/*:$HBASE_CP:$EXTRA_CLASSPATH
+ else
+ # assume Hadoop/HBase libs are included via EXTRA_CLASSPATH
+ echo "WARN: could not find Hadoop and HBase libraries"
+ CP=$COMP_HOME/phoenix-assembly/target/*:$EXTRA_CLASSPATH
+ fi
+
+ # Setup classpaths.
+ if [ -n "$CLASSPATH" ]; then
+ CLASSPATH=$CLASSPATH:$CP
+ else
+ CLASSPATH=$CP
+ fi
+
+ export CLASSPATH
+}
+
+# Attempts to find JAVA in few ways.
+set_java ()
+{
+ # Determine the Java command to use to start the JVM.
+ if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ export JAVA="$JAVA_HOME/jre/sh/java"
+ else
+ export JAVA="$JAVA_HOME/bin/java"
+ fi
+ if [ ! -x "$JAVA" ] ; then
+ echo "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation." >&2
+ exit 1
+ fi
+ else
+ export JAVA="java"
+ which java >/dev/null 2>&1 || { echo "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation." >&2 ; exit 1; }
+ fi
+}
+
+# checks if there exists a PID that is already running. return 0 idempotently
+check_before_start()
+{
+ if [ ! -d "$PID_DIR" ]; then
+ mkdir -p "$PID_DIR"
+ fi
+ if [ -f $pid ]; then
+ if kill -0 `cat $pid` > /dev/null 2>&1; then
+ #echo "$APP $SERVICE running as process `cat $pid`. Stop it first."
+ echo "$APP running as process `cat $pid`. Stop it first."
+ exit 0
+ fi
+ fi
+}
+
+# Set Niceness
+if [ "$NICENESS" = "" ]; then
+ export NICENESS=0
+fi
+
+start() {
+ # Setup classpaths.
+ set_classpath $APP_HOME
+
+ # sets the JAVA variable.
+ set_java
+
+ check_before_start
+
+ echo "`date` Starting $APP service on `hostname`"
+ echo "`date` Starting $APP service on `hostname`" >> $loglog
+ echo "`ulimit -a`" >> $loglog 2>&1
+
+ export MAIN_CLASS="co.cask.tephra.TransactionServiceMain"
+ echo "Running class $MAIN_CLASS"
+ echo "Command: " "$JAVA" $OPTS -cp $CLASSPATH $JAVA_HEAPMAX $MAIN_CLASS >>$loglog
+ nohup nice -n $NICENESS "$JAVA" $OPTS -cp $CLASSPATH $JAVA_HEAPMAX ${MAIN_CLASS} </dev/null >>$loglog 2>&1 &
+ echo $! >$pid
+}
+
+stop() {
+ if [ -f $pid ]; then
+ pidToKill=`cat $pid`
+ # kill -0 == see if the PID exists
+ if kill -0 $pidToKill > /dev/null 2>&1; then
+ echo -n stopping $command
+ echo "`date` Terminating $command" >> $loglog
+ kill $pidToKill > /dev/null 2>&1
+ while kill -0 $pidToKill > /dev/null 2>&1;
+ do
+ echo -n "."
+ sleep 1;
+ done
+ rm $pid
+ echo
+ else
+ retval=$?
+ echo nothing to stop because kill -0 of pid $pidToKill failed with status $retval
+ fi
+ rm -f $pid
+ else
+ echo nothing to stop because no pid file $pid
+ fi
+}
+
+restart() {
+ stop
+ start
+}
+
+condrestart(){
+ case "$PLATFORM" in
+ "RHEL")
+ rh_status > /dev/null 2>&1
+ retval=$?
+ ;;
+ "UBUNTU")
+ ub_status > /dev/null 2>&1
+ retval=$?
+ ;;
+ "UNSUPPORTED")
+ echo "condrestart is not supported on platform"
+ exit 1
+ ;;
+ esac
+
+ if [[ $retval -eq 0 ]]; then
+ restart
+ fi
+}
+
+rh_status() {
+ echo "checking status"
+ # call sourced status function
+ status -p $pid
+}
+
+ub_status() {
+ echo "checking status"
+ # call sourced status function
+ status_of_proc -p $pid "$0" "$APP"
+}
+
+# Executes a specific class' main method with the classpath and environment setup
+run() {
+ classname=$1
+ shift
+ if [ -z "$classname" ]; then
+ echo "ERROR: No classname was given!"
+ echo "Usage: $0 run <fully qualified classname> [arguments]"
+ exit 1
+ fi
+ # Setup classpaths.
+ set_classpath $APP_HOME
+
+ # sets the JAVA variable.
+ set_java
+
+ echo "Running class $classname"
+ "$JAVA" $OPTS -cp $CLASSPATH $JAVA_HEAPMAX $classname $@
+}
+
+case "$1" in
+ start)
+ $1
+ ;;
+
+ stop)
+ $1
+ ;;
+
+ restart)
+ $1
+ ;;
+
+ condrestart)
+ $1
+ ;;
+
+ status)
+ case "$PLATFORM" in
+ "RHEL")
+ rh_status
+ ;;
+ "UBUNTU")
+ ub_status
+ ;;
+ "UNSUPPORTED")
+ echo "status is not supported on platform"
+ exit 1
+ ;;
+ esac
+ ;;
+
+ classpath)
+ set_classpath $APP_HOME
+ set_java
+ echo $CLASSPATH
+ ;;
+
+ run)
+ shift
+ run $@
+ ;;
+
+ *)
+ echo "Usage: $0 {start|stop|restart|status|run}"
+ exit 1
+ ;;
+
+
+esac
+exit $?
+
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/bin/tephra-env.sh
----------------------------------------------------------------------
diff --git a/bin/tephra-env.sh b/bin/tephra-env.sh
new file mode 100644
index 0000000..6906b44
--- /dev/null
+++ b/bin/tephra-env.sh
@@ -0,0 +1,55 @@
+#!/bin/sh
+
+#
+# Copyright © 2014 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+
+#
+# Common environment settings for Tephra.
+# Uncomment the lines below, where needed, and modify to adapt to your environment.
+#
+
+# A string representing this instance of the Tephra server. $USER by default.
+export IDENT_STRING=$USER
+
+# Where log files are stored. /var/log by default.
+export LOG_DIR=/tmp/tephra-$IDENT_STRING
+
+# The directory where pid files are stored. /var/run by default.
+export PID_DIR=/tmp
+
+# Add any extra classes to the classpath
+# export EXTRA_CLASSPATH
+
+# Set the JVM heap size
+# export JAVA_HEAPMAX=-Xmx2048m
+
+# Additional runtime options
+#
+# GC logging options.
+# Uncomment the following two lines, making any desired changes, to enable GC logging output
+# export GC_LOG_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:server-gc.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=2 -XX:GCLogFileSize=50M"
+# export OPTS="$OPTS $GC_LOG_OPTS"
+#
+# JMX options.
+# Uncomment the following two lines, making any desired changes, to enable remote JMX connectivity
+# export JMX_OPTS="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=13001"
+# export OPTS="$OPTS $JMX_OPTS"
+
+# Extra Java runtime options.
+# Below are what we set by default. May only work with SUN JVM.
+# For more on why as well as other possible settings,
+# see http://wiki.apache.org/hadoop/PerformanceTuning
+export OPTS="$OPTS -XX:+UseConcMarkSweepGC"
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-assembly/src/build/client.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/client.xml b/phoenix-assembly/src/build/client.xml
index 0e1e1f6..c1fe861 100644
--- a/phoenix-assembly/src/build/client.xml
+++ b/phoenix-assembly/src/build/client.xml
@@ -59,6 +59,10 @@
<include>org.apache.htrace:htrace-core</include>
<include>io.netty:netty-all</include>
<include>commons-codec:commons-codec</include>
+ <include>co.cask.tephra:tephra*</include>
+ <include>org.apache.twill:twill*</include>
+ <include>org.apache.thrift:*</include>
+ <include>com.google.code.gson:gson*</include>
</includes>
</dependencySet>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-assembly/src/build/server-with-txns.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/server-with-txns.xml b/phoenix-assembly/src/build/server-with-txns.xml
new file mode 100644
index 0000000..2808a8c
--- /dev/null
+++ b/phoenix-assembly/src/build/server-with-txns.xml
@@ -0,0 +1,48 @@
+<?xml version='1.0'?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <!-- build the phoenix server side jar, that includes phoenix-hadoopX-compat, phoenix-hadoop-compat and antlr -->
+ <id>server-with-txns</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/</outputDirectory>
+ <unpack>true</unpack>
+ <includes>
+ <include>org.apache.phoenix:phoenix-core</include>
+ <include>org.iq80.snappy:snappy</include>
+ <include>org.antlr:antlr*</include>
+ <include>co.cask.tephra:tephra*</include>
+ <include>com.google.code.gson:gson</include>
+ <include>com.google.inject.extensions:guice-assistedinject</include>
+ <include>it.unimi.dsi:fastutil</include>
+ <include>io.dropwizard.metrics:metrics-core</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-assembly/src/build/server-without-antlr.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/server-without-antlr.xml b/phoenix-assembly/src/build/server-without-antlr.xml
new file mode 100644
index 0000000..33cf0b7
--- /dev/null
+++ b/phoenix-assembly/src/build/server-without-antlr.xml
@@ -0,0 +1,46 @@
+<?xml version='1.0'?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <!-- build the phoenix server side jar, that includes phoenix-hadoopX-compat and phoenix-hadoop-compat. -->
+ <id>server-without-antlr</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/</outputDirectory>
+ <unpack>true</unpack>
+ <includes>
+ <include>org.apache.phoenix:phoenix-core</include>
+ <include>org.iq80.snappy:snappy</include>
+ <include>co.cask.tephra:tephra*</include>
+ <include>com.google.code.gson:gson</include>
+ <include>org.jruby.joni:joni</include>
+ <include>org.jruby.jcodings:jcodings</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-assembly/src/build/server.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/server.xml b/phoenix-assembly/src/build/server.xml
index 78a4b1f..141f97b 100644
--- a/phoenix-assembly/src/build/server.xml
+++ b/phoenix-assembly/src/build/server.xml
@@ -36,17 +36,13 @@
<includes>
<include>org.apache.phoenix:phoenix-core</include>
<include>org.iq80.snappy:snappy</include>
+ <include>org.antlr:antlr*</include>
+ <include>co.cask.tephra:tephra*</include>
+ <include>com.google.code.gson:gson</include>
<include>org.jruby.joni:joni</include>
<include>org.jruby.jcodings:jcodings</include>
<include>joda-time:joda-time</include>
</includes>
</dependencySet>
- <dependencySet>
- <unpack>true</unpack>
- <outputDirectory>/</outputDirectory>
- <includes>
- <include>org.antlr:antlr*</include>
- </includes>
- </dependencySet>
</dependencySets>
</assembly>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index 2bb7466..5c73ed4 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -224,6 +224,30 @@
</build>
<dependencies>
+ <!-- Transaction dependencies -->
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-api</artifactId>
+ <version>${tephra.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <version>${tephra.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <type>test-jar</type>
+ <version>${tephra.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-hbase-compat-1.1</artifactId>
+ <version>${tephra.version}</version>
+ </dependency>
+
<!-- Make sure we have all the antlr dependencies -->
<dependency>
<groupId>org.antlr</groupId>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
index bee8d21..d4f7c16 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
@@ -35,8 +35,8 @@ import org.apache.phoenix.query.QueryServices;
public class ConnectionQueryServicesTestImpl extends ConnectionQueryServicesImpl {
protected int NUM_SLAVES_BASE = 1; // number of slaves for the cluster
- public ConnectionQueryServicesTestImpl(QueryServices services, ConnectionInfo info) throws SQLException {
- super(services, info, null);
+ public ConnectionQueryServicesTestImpl(QueryServices services, ConnectionInfo info, Properties props) throws SQLException {
+ super(services, info, props);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
index 88a8215..6191a4f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
@@ -133,7 +133,7 @@ public class StatsCollectorWithSplitsAndMultiCFIT extends StatsCollectorAbstract
Collection<GuidePostsInfo> infos = TestUtil.getGuidePostsList(conn, STATS_TEST_TABLE_NAME_NEW);
long[] rowCountArr = new long[]{25, 20, 25, 25};
// CF A alone has double the bytecount because it has column qualifier A and column qualifier _0
- long[] byteCountArr = new long[]{12120, 5540, 6652, 6652};
+ long[] byteCountArr = new long[]{12144, 5540, 6652, 6652};
int i = 0;
for(GuidePostsInfo info : infos) {
assertRowCountAndByteCount(info, rowCountArr[i], byteCountArr[i]);
@@ -148,8 +148,8 @@ public class StatsCollectorWithSplitsAndMultiCFIT extends StatsCollectorAbstract
}
protected void assertRowCountAndByteCount(GuidePostsInfo info, long rowCount, long byteCount) {
- assertEquals(info.getRowCount(), rowCount);
- assertEquals(info.getByteCount(), byteCount);
+ assertEquals("Row count does not match ", rowCount, info.getRowCount());
+ assertEquals("Byte count does not match ", byteCount, info.getByteCount());
}
@Test
@@ -176,7 +176,7 @@ public class StatsCollectorWithSplitsAndMultiCFIT extends StatsCollectorAbstract
Collection<GuidePostsInfo> infos = TestUtil.getGuidePostsList(conn, STATS_TEST_TABLE_NAME);
for (GuidePostsInfo info : infos) {
assertEquals(20, info.getRowCount());
- assertEquals(11020, info.getByteCount());
+ assertEquals(11040, info.getByteCount());
break;
}
List<KeyRange> keyRanges = getAllSplits(conn, STATS_TEST_TABLE_NAME);
@@ -269,7 +269,7 @@ public class StatsCollectorWithSplitsAndMultiCFIT extends StatsCollectorAbstract
infos = TestUtil.getGuidePostsList(conn, STATS_TEST_TABLE_NAME);
for (GuidePostsInfo info : infos) {
assertEquals(20, info.getRowCount());
- assertEquals(9918, info.getByteCount());
+ assertEquals(9936, info.getByteCount());
break;
}
conn.close();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java
index 240d335..b03037c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java
@@ -48,17 +48,28 @@ public class ImmutableIndexIT extends BaseHBaseManagedTimeIT {
private final boolean localIndex;
private final String tableDDLOptions;
+ private final String tableName;
+ private final String indexName;
+ private final String fullTableName;
+ private final String fullIndexName;
- public ImmutableIndexIT(boolean localIndex) {
+ public ImmutableIndexIT(boolean localIndex, boolean transactional) {
this.localIndex = localIndex;
StringBuilder optionBuilder = new StringBuilder("IMMUTABLE_ROWS=true");
+ if (transactional) {
+ optionBuilder.append(", TRANSACTIONAL=true");
+ }
this.tableDDLOptions = optionBuilder.toString();
+ this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + ( transactional ? "_TXN" : "");
+ this.indexName = "IDX" + ( transactional ? "_TXN" : "");
+ this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ this.fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
}
- @Parameters(name="localIndex = {0}")
+ @Parameters(name="localIndex = {0} , transactional = {1}")
public static Collection<Boolean[]> data() {
return Arrays.asList(new Boolean[][] {
- { false}, { true }
+ { false, false }, { false, true }, { true, false }, { true, true }
});
}
@@ -67,11 +78,6 @@ public class ImmutableIndexIT extends BaseHBaseManagedTimeIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
Statement stmt = conn.createStatement();
stmt.execute(ddl);
@@ -102,4 +108,4 @@ public class ImmutableIndexIT extends BaseHBaseManagedTimeIT {
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
index e3d1db7..7da0d85 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
@@ -38,6 +38,7 @@ import org.apache.phoenix.util.DateUtil;
import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.TestUtil;
import org.junit.Test;
public class IndexExpressionIT extends BaseHBaseManagedTimeIT {
@@ -75,7 +76,7 @@ public class IndexExpressionIT extends BaseHBaseManagedTimeIT {
stmt.setInt(3, i);
stmt.setLong(4, i);
stmt.setBigDecimal(5, new BigDecimal(i*0.5d));
- Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * NUM_MILLIS_IN_DAY);
+ Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * TestUtil.NUM_MILLIS_IN_DAY);
stmt.setDate(6, date);
stmt.setString(7, "a.varchar" + String.valueOf(i));
stmt.setString(8, "a.char" + String.valueOf(i));
@@ -98,7 +99,7 @@ public class IndexExpressionIT extends BaseHBaseManagedTimeIT {
+ "_A.VARCHAR" + String.valueOf(i) + "_" + StringUtils.rightPad("B.CHAR" + String.valueOf(i), 10, ' '),
rs.getString(1));
assertEquals(i * 3, rs.getInt(2));
- Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i) * NUM_MILLIS_IN_DAY);
+ Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i) * TestUtil.NUM_MILLIS_IN_DAY);
assertEquals(date, rs.getDate(3));
assertEquals(date, rs.getDate(4));
assertEquals(date, rs.getDate(5));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
index 695225f..d2c6267 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
@@ -33,7 +33,6 @@ import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -44,20 +43,33 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
private final boolean localIndex;
private final String tableDDLOptions;
+ private final String tableName;
+ private final String indexName;
+ private final String fullTableName;
+ private final String fullIndexName;
- public IndexIT(boolean localIndex, boolean mutable) {
+ public IndexIT(boolean localIndex, boolean mutable, boolean transactional) {
this.localIndex = localIndex;
StringBuilder optionBuilder = new StringBuilder();
if (!mutable)
optionBuilder.append(" IMMUTABLE_ROWS=true ");
+ if (transactional) {
+ if (!(optionBuilder.length()==0))
+ optionBuilder.append(",");
+ optionBuilder.append(" TRANSACTIONAL=true ");
+ }
this.tableDDLOptions = optionBuilder.toString();
+ this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + ( transactional ? "_TXN" : "");
+ this.indexName = "IDX" + ( transactional ? "_TXN" : "");
+ this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ this.fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
}
- @Parameters(name="localIndex = {0} , mutable = {1}")
+ @Parameters(name="localIndex = {0} , mutable = {1} , transactional = {2}")
public static Collection<Boolean[]> data() {
return Arrays.asList(new Boolean[][] {
- { false, false }, { false, true },
- { true, false }, { true, true }
+ { false, false, false }, { false, false, true }, { false, true, false }, { false, true, true },
+ { true, false, false }, { true, false, true }, { true, true, false }, { true, true, true }
});
}
@@ -66,10 +78,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
Statement stmt = conn.createStatement();
stmt.execute(ddl);
@@ -122,16 +130,10 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
}
@Test
- @Ignore("Failing due to zero byte incorrectly being stripped from row key") // FIXME: fixed in master, so remove this ignore tag when merged.
public void testDeleteFromAllPKColumnIndex() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
Statement stmt = conn.createStatement();
stmt.execute(ddl);
@@ -185,11 +187,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
@Test
public void testDeleteFromNonPKColumnIndex() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
@@ -242,10 +239,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
Statement stmt = conn.createStatement();
stmt.execute(ddl);
@@ -264,10 +257,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
Statement stmt = conn.createStatement();
stmt.execute(ddl);
@@ -291,10 +280,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
Statement stmt = conn.createStatement();
stmt.execute(ddl);
@@ -319,11 +304,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
String ddl ="CREATE TABLE " + fullTableName
+ " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) DEFAULT_COLUMN_FAMILY='A'" + (!tableDDLOptions.isEmpty() ? "," + tableDDLOptions : "");
Statement stmt = conn.createStatement();
@@ -361,11 +341,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
Date date = new Date(System.currentTimeMillis());
createMultiCFTestTable(fullTableName, tableDDLOptions);
@@ -423,12 +398,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
String ddl = "CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) " + tableDDLOptions;
conn.createStatement().execute(ddl);
query = "SELECT * FROM " + fullTableName;
@@ -504,12 +473,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
String ddl = "CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, a.v1 VARCHAR, a.v2 VARCHAR, b.v1 VARCHAR) " + tableDDLOptions;
conn.createStatement().execute(ddl);
query = "SELECT * FROM " + fullTableName;
@@ -567,12 +530,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
// make sure that the tables are empty, but reachable
conn.createStatement().execute(
"CREATE TABLE " + fullTableName
@@ -629,34 +586,29 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
@Test
public void testMultipleUpdatesAcrossRegions() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String testTable = fullTableName+"_MULTIPLE_UPDATES";
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
// make sure that the tables are empty, but reachable
conn.createStatement().execute(
- "CREATE TABLE " + fullTableName
+ "CREATE TABLE " + testTable
+ " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) " + HTableDescriptor.MAX_FILESIZE + "=1, " + HTableDescriptor.MEMSTORE_FLUSHSIZE + "=1 "
+ (!tableDDLOptions.isEmpty() ? "," + tableDDLOptions : "") + "SPLIT ON ('b')");
- query = "SELECT * FROM " + fullTableName;
+ query = "SELECT * FROM " + testTable;
rs = conn.createStatement().executeQuery(query);
assertFalse(rs.next());
conn.createStatement().execute(
- "CREATE " + (localIndex ? "LOCAL " : "") + "INDEX " + indexName + " ON " + fullTableName + " (v1, v2)");
+ "CREATE " + (localIndex ? "LOCAL " : "") + "INDEX " + indexName + " ON " + testTable + " (v1, v2)");
query = "SELECT * FROM " + fullIndexName;
rs = conn.createStatement().executeQuery(query);
assertFalse(rs.next());
// load some data into the table
PreparedStatement stmt =
- conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
+ conn.prepareStatement("UPSERT INTO " + testTable + " VALUES(?,?,?)");
stmt.setString(1, "a");
stmt.setString(2, "x");
stmt.setString(3, "1");
@@ -688,10 +640,10 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
assertEquals("c", rs.getString(3));
assertFalse(rs.next());
- query = "SELECT * FROM " + fullTableName;
+ query = "SELECT * FROM " + testTable;
rs = conn.createStatement().executeQuery("EXPLAIN " + query);
if (localIndex) {
- assertEquals("CLIENT PARALLEL 2-WAY RANGE SCAN OVER _LOCAL_IDX_" + fullTableName+" [-32768]\n"
+ assertEquals("CLIENT PARALLEL 2-WAY RANGE SCAN OVER _LOCAL_IDX_" + testTable+" [-32768]\n"
+ " SERVER FILTER BY FIRST KEY ONLY\n"
+ "CLIENT MERGE SORT",
QueryUtil.getExplainPlan(rs));
@@ -726,12 +678,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
conn.createStatement().execute("CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, \"V1\" VARCHAR, \"v2\" VARCHAR)"+tableDDLOptions);
query = "SELECT * FROM "+fullTableName;
rs = conn.createStatement().executeQuery(query);
@@ -815,11 +761,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
-
String ddl = "CREATE TABLE " + fullTableName +" (PK1 CHAR(2) NOT NULL PRIMARY KEY, CF1.COL1 BIGINT) " + tableDDLOptions;
conn.createStatement().execute(ddl);
ddl = "CREATE " + (localIndex ? "LOCAL " : "") + "INDEX " + indexName + " ON " + fullTableName + "(COL1)";
@@ -838,11 +779,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
Date date = new Date(System.currentTimeMillis());
createMultiCFTestTable(fullTableName, tableDDLOptions);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index 364b358..39c90be 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -26,11 +26,13 @@ import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Properties;
@@ -69,11 +71,15 @@ import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.StringUtil;
+import org.apache.phoenix.util.TestUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
/**
*
* Test for failure of region server to write to index table.
@@ -85,18 +91,30 @@ import org.junit.experimental.categories.Category;
*/
@Category(NeedsOwnMiniClusterTest.class)
+@RunWith(Parameterized.class)
public class MutableIndexFailureIT extends BaseTest {
private static final int NUM_SLAVES = 4;
- private static String url;
private static PhoenixTestDriver driver;
private static HBaseTestingUtility util;
private Timer scheduleTimer;
- private static final String SCHEMA_NAME = "S";
- private static final String INDEX_TABLE_NAME = "I";
- private static final String DATA_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, "T");
- private static final String INDEX_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, "I");
+ private String tableName;
+ private String indexName;
+ private String fullTableName;
+ private String fullIndexName;
+ private boolean transactional;
+ private final String tableDDLOptions;
+
+ public MutableIndexFailureIT(boolean transactional) {
+ this.transactional = transactional;
+ this.tableDDLOptions = transactional ? " TRANSACTIONAL=true " : "";
+ this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME;
+ this.indexName = "IDX";
+ this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ this.fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
+ }
+
@Before
public void doSetup() throws Exception {
Configuration conf = HBaseConfiguration.create();
@@ -114,8 +132,15 @@ public class MutableIndexFailureIT extends BaseTest {
url = JDBC_PROTOCOL + JDBC_PROTOCOL_SEPARATOR + LOCALHOST + JDBC_PROTOCOL_SEPARATOR + clientPort
+ JDBC_PROTOCOL_TERMINATOR + PHOENIX_TEST_DRIVER_URL_PARAM;
driver = initAndRegisterDriver(url, ReadOnlyProps.EMPTY_PROPS);
+ clusterInitialized = true;
+ setupTxManager();
}
-
+
+ @Parameters(name = "transactional = {0}")
+ public static Collection<Boolean[]> data() {
+ return Arrays.asList(new Boolean[][] { { false }, { true } });
+ }
+
@After
public void tearDown() throws Exception {
try {
@@ -135,230 +160,279 @@ public class MutableIndexFailureIT extends BaseTest {
@Ignore("See PHOENIX-2331")
@Test(timeout=300000)
public void testWriteFailureDisablesLocalIndex() throws Exception {
- testWriteFailureDisablesIndex(true);
+ helpTestWriteFailureDisablesIndex(true);
}
@Ignore("See PHOENIX-2332")
@Test(timeout=300000)
public void testWriteFailureDisablesIndex() throws Exception {
- testWriteFailureDisablesIndex(false);
+ helpTestWriteFailureDisablesIndex(false);
}
- public void testWriteFailureDisablesIndex(boolean localIndex) throws Exception {
- String query;
- ResultSet rs;
-
+ public void helpTestWriteFailureDisablesIndex(boolean localIndex) throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = driver.connect(url, props);
- conn.setAutoCommit(false);
- conn.createStatement().execute(
- "CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
- query = "SELECT * FROM " + DATA_TABLE_FULL_NAME;
- rs = conn.createStatement().executeQuery(query);
- assertFalse(rs.next());
-
- if(localIndex) {
+ try (Connection conn = driver.connect(url, props)) {
+ String query;
+ ResultSet rs;
+ conn.setAutoCommit(false);
conn.createStatement().execute(
- "CREATE LOCAL INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v1) INCLUDE (v2)");
- conn.createStatement().execute(
- "CREATE LOCAL INDEX " + INDEX_TABLE_NAME+ "_2" + " ON " + DATA_TABLE_FULL_NAME + " (v2) INCLUDE (v1)");
- } else {
- conn.createStatement().execute(
- "CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v1) INCLUDE (v2)");
- }
+ "CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) "+tableDDLOptions);
+ query = "SELECT * FROM " + fullTableName;
+ rs = conn.createStatement().executeQuery(query);
+ assertFalse(rs.next());
+
+ if(localIndex) {
+ conn.createStatement().execute(
+ "CREATE LOCAL INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE (v2)");
+ conn.createStatement().execute(
+ "CREATE LOCAL INDEX " + indexName+ "_2" + " ON " + fullTableName + " (v2) INCLUDE (v1)");
+ } else {
+ conn.createStatement().execute(
+ "CREATE INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE (v2)");
+ }
+
+ query = "SELECT * FROM " + fullIndexName;
+ rs = conn.createStatement().executeQuery(query);
+ assertFalse(rs.next());
+
+ // Verify the metadata for index is correct.
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
+ new String[] { PTableType.INDEX.toString() });
+ assertTrue(rs.next());
+ assertEquals(indexName, rs.getString(3));
+ assertEquals(PIndexState.ACTIVE.toString(), rs.getString("INDEX_STATE"));
+ assertFalse(rs.next());
- query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME;
- rs = conn.createStatement().executeQuery(query);
- assertFalse(rs.next());
-
- // Verify the metadata for index is correct.
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(SCHEMA_NAME), INDEX_TABLE_NAME,
- new String[] { PTableType.INDEX.toString() });
- assertTrue(rs.next());
- assertEquals(INDEX_TABLE_NAME, rs.getString(3));
- assertEquals(PIndexState.ACTIVE.toString(), rs.getString("INDEX_STATE"));
- assertFalse(rs.next());
-
- PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)");
- stmt.setString(1, "a");
- stmt.setString(2, "x");
- stmt.setString(3, "1");
- stmt.execute();
- conn.commit();
-
- TableName indexTable =
- TableName.valueOf(localIndex ? MetaDataUtil
- .getLocalIndexTableName(DATA_TABLE_FULL_NAME) : INDEX_TABLE_FULL_NAME);
- HBaseAdmin admin = this.util.getHBaseAdmin();
- HTableDescriptor indexTableDesc = admin.getTableDescriptor(indexTable);
- try{
- admin.disableTable(indexTable);
- admin.deleteTable(indexTable);
- } catch (TableNotFoundException ignore) {}
-
- stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)");
- stmt.setString(1, "a2");
- stmt.setString(2, "x2");
- stmt.setString(3, "2");
- stmt.execute();
- try {
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
+ stmt.setString(1, "a");
+ stmt.setString(2, "x");
+ stmt.setString(3, "1");
+ stmt.execute();
conn.commit();
- } catch (SQLException e) {}
-
- // Verify the metadata for index is correct.
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(SCHEMA_NAME), INDEX_TABLE_NAME,
- new String[] { PTableType.INDEX.toString() });
- assertTrue(rs.next());
- assertEquals(INDEX_TABLE_NAME, rs.getString(3));
- assertEquals(PIndexState.DISABLE.toString(), rs.getString("INDEX_STATE"));
- assertFalse(rs.next());
- if(localIndex) {
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(SCHEMA_NAME), INDEX_TABLE_NAME+"_2",
- new String[] { PTableType.INDEX.toString() });
+
+ TableName indexTable =
+ TableName.valueOf(localIndex ? MetaDataUtil
+ .getLocalIndexTableName(fullTableName) : fullIndexName);
+ HBaseAdmin admin = this.util.getHBaseAdmin();
+ HTableDescriptor indexTableDesc = admin.getTableDescriptor(indexTable);
+ try{
+ admin.disableTable(indexTable);
+ admin.deleteTable(indexTable);
+ } catch (TableNotFoundException ignore) {}
+
+ stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
+ stmt.setString(1, "a2");
+ stmt.setString(2, "x2");
+ stmt.setString(3, "2");
+ stmt.execute();
+ if (transactional) {
+ try {
+ conn.commit();
+ fail();
+ } catch (SQLException e) {
+ conn.rollback();
+ }
+ }
+ else {
+ conn.commit();
+ }
+
+ // Verify the metadata for index is correct.
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
+ new String[] { PTableType.INDEX.toString() });
assertTrue(rs.next());
- assertEquals(INDEX_TABLE_NAME+"_2", rs.getString(3));
- assertEquals(PIndexState.DISABLE.toString(), rs.getString("INDEX_STATE"));
+ assertEquals(indexName, rs.getString(3));
+ // the index is only disabled for non-txn tables upon index table write failure
+ PIndexState indexState = transactional ? PIndexState.ACTIVE : PIndexState.DISABLE;
+ assertEquals(indexState.toString(), rs.getString("INDEX_STATE"));
assertFalse(rs.next());
- }
-
- // Verify UPSERT on data table still work after index is disabled
- stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)");
- stmt.setString(1, "a3");
- stmt.setString(2, "x3");
- stmt.setString(3, "3");
- stmt.execute();
- conn.commit();
-
- query = "SELECT v2 FROM " + DATA_TABLE_FULL_NAME + " where v1='x3'";
- rs = conn.createStatement().executeQuery("EXPLAIN " + query);
- assertTrue(QueryUtil.getExplainPlan(rs).contains("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + DATA_TABLE_FULL_NAME));
- rs = conn.createStatement().executeQuery(query);
- assertTrue(rs.next());
-
- // recreate index table
- admin.createTable(indexTableDesc);
- do {
- Thread.sleep(15 * 1000); // sleep 15 secs
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(SCHEMA_NAME), INDEX_TABLE_NAME,
- new String[] { PTableType.INDEX.toString() });
- assertTrue(rs.next());
- if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
- break;
- }
- if(localIndex) {
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(SCHEMA_NAME), INDEX_TABLE_NAME+"_2",
+ if(localIndex) {
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName + "_2",
+ new String[] { PTableType.INDEX.toString() });
+ assertTrue(rs.next());
+ assertEquals(indexName + "_2", rs.getString(3));
+ assertEquals(indexState.toString(), rs.getString("INDEX_STATE"));
+ assertFalse(rs.next());
+ }
+
+ // if the table is transactional the write to the index table will fail because the
+ // index has not been disabled
+ if (!transactional) {
+ // Verify UPSERT on data table still work after index is disabled
+ stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
+ stmt.setString(1, "a3");
+ stmt.setString(2, "x3");
+ stmt.setString(3, "3");
+ stmt.execute();
+ conn.commit();
+ }
+
+ if (transactional) {
+ // if the table was transactional there should be 1 row (written before the index
+ // was disabled)
+ query = "SELECT /*+ NO_INDEX */ v2 FROM " + fullTableName;
+ rs = conn.createStatement().executeQuery("EXPLAIN " + query);
+ String expectedPlan =
+ "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + fullTableName;
+ assertEquals(expectedPlan, QueryUtil.getExplainPlan(rs));
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("1", rs.getString(1));
+ assertFalse(rs.next());
+ } else {
+ // if the table was not transactional there should be three rows (all writes to data
+ // table should succeed)
+ query = "SELECT v2 FROM " + fullTableName;
+ rs = conn.createStatement().executeQuery("EXPLAIN " + query);
+ String expectedPlan =
+ "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + fullTableName;
+ assertEquals(expectedPlan, QueryUtil.getExplainPlan(rs));
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("1", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("2", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("3", rs.getString(1));
+ assertFalse(rs.next());
+ }
+
+ // recreate index table
+ admin.createTable(indexTableDesc);
+ do {
+ Thread.sleep(15 * 1000); // sleep 15 secs
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
new String[] { PTableType.INDEX.toString() });
assertTrue(rs.next());
if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
break;
}
- }
- } while(true);
-
- // verify index table has data
- query = "SELECT count(1) FROM " + INDEX_TABLE_FULL_NAME;
- rs = conn.createStatement().executeQuery(query);
- assertTrue(rs.next());
-
- // using 2 here because we only partially build index from where we failed and the oldest
- // index row has been deleted when we dropped the index table during test.
- assertEquals(2, rs.getInt(1));
- }
-
- @Test(timeout=300000)
- public void testWriteFailureWithRegionServerDown() throws Exception {
- String query;
- ResultSet rs;
-
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = driver.connect(url, props);
- conn.setAutoCommit(false);
- conn.createStatement().execute(
- "CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
- query = "SELECT * FROM " + DATA_TABLE_FULL_NAME;
- rs = conn.createStatement().executeQuery(query);
- assertFalse(rs.next());
+ if(localIndex) {
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName + "_2",
+ new String[] { PTableType.INDEX.toString() });
+ assertTrue(rs.next());
+ if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
+ break;
+ }
+ }
+ } while(true);
+
+ // Verify UPSERT on data table still work after index table is recreated
+ stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
+ stmt.setString(1, "a4");
+ stmt.setString(2, "x4");
+ stmt.setString(3, "4");
+ stmt.execute();
+ conn.commit();
- conn.createStatement().execute(
- "CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v1) INCLUDE (v2)");
- query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME;
- rs = conn.createStatement().executeQuery(query);
- assertFalse(rs.next());
+ // verify index table has data
+ query = "SELECT count(1) FROM " + fullIndexName;
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
- // Verify the metadata for index is correct.
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(SCHEMA_NAME), INDEX_TABLE_NAME,
- new String[] { PTableType.INDEX.toString() });
- assertTrue(rs.next());
- assertEquals(INDEX_TABLE_NAME, rs.getString(3));
- assertEquals(PIndexState.ACTIVE.toString(), rs.getString("INDEX_STATE"));
- assertFalse(rs.next());
-
- PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)");
- stmt.setString(1, "a");
- stmt.setString(2, "x");
- stmt.setString(3, "1");
- stmt.execute();
- conn.commit();
-
- // find a RS which doesn't has CATALOG table
- TableName catalogTable = TableName.valueOf("SYSTEM.CATALOG");
- TableName indexTable = TableName.valueOf(INDEX_TABLE_FULL_NAME);
- final HBaseCluster cluster = this.util.getHBaseCluster();
- Collection<ServerName> rss = cluster.getClusterStatus().getServers();
- HBaseAdmin admin = this.util.getHBaseAdmin();
- List<HRegionInfo> regions = admin.getTableRegions(catalogTable);
- ServerName catalogRS = cluster.getServerHoldingRegion(regions.get(0).getTable(),
- regions.get(0).getRegionName());
- ServerName metaRS = cluster.getServerHoldingMeta();
- ServerName rsToBeKilled = null;
-
- // find first RS isn't holding META or CATALOG table
- for(ServerName curRS : rss) {
- if(!curRS.equals(catalogRS) && !metaRS.equals(curRS)) {
- rsToBeKilled = curRS;
- break;
+ // for txn tables there will be only one row in the index (a4)
+ // for non txn tables there will be three rows because we only partially build index
+ // from where we failed and the oldest
+ // index row has been deleted when we dropped the index table during test
+ assertEquals(transactional ? 1 : 3, rs.getInt(1));
}
}
- assertTrue(rsToBeKilled != null);
-
- regions = admin.getTableRegions(indexTable);
- final HRegionInfo indexRegion = regions.get(0);
- final ServerName dstRS = rsToBeKilled;
- admin.move(indexRegion.getEncodedNameAsBytes(), Bytes.toBytes(rsToBeKilled.getServerName()));
- this.util.waitFor(30000, 200, new Waiter.Predicate<Exception>() {
- @Override
- public boolean evaluate() throws Exception {
- ServerName sn = cluster.getServerHoldingRegion(indexRegion.getTable(),
- indexRegion.getRegionName());
- return (sn != null && sn.equals(dstRS));
- }
- });
-
- // use timer sending updates in every 10ms
- this.scheduleTimer = new Timer(true);
- this.scheduleTimer.schedule(new SendingUpdatesScheduleTask(conn), 0, 10);
- // let timer sending some updates
- Thread.sleep(100);
- // kill RS hosting index table
- this.util.getHBaseCluster().killRegionServer(rsToBeKilled);
-
- // wait for index table completes recovery
- this.util.waitUntilAllRegionsAssigned(indexTable);
+ @Test(timeout=300000)
+ public void testWriteFailureWithRegionServerDown() throws Exception {
+ String query;
+ ResultSet rs;
+
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = driver.connect(url, props);) {
+ conn.setAutoCommit(false);
+ conn.createStatement().execute(
+ "CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) "+tableDDLOptions);
+ query = "SELECT * FROM " + fullTableName;
+ rs = conn.createStatement().executeQuery(query);
+ assertFalse(rs.next());
- // Verify the metadata for index is correct.
- do {
- Thread.sleep(15 * 1000); // sleep 15 secs
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(SCHEMA_NAME), INDEX_TABLE_NAME,
- new String[] { PTableType.INDEX.toString() });
- assertTrue(rs.next());
- if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
- break;
- }
- } while(true);
- this.scheduleTimer.cancel();
+ conn.createStatement().execute(
+ "CREATE INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE (v2)");
+ query = "SELECT * FROM " + fullIndexName;
+ rs = conn.createStatement().executeQuery(query);
+ assertFalse(rs.next());
- assertEquals(cluster.getClusterStatus().getDeadServers(), 1);
+ // Verify the metadata for index is correct.
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
+ new String[] { PTableType.INDEX.toString() });
+ assertTrue(rs.next());
+ assertEquals(indexName, rs.getString(3));
+ assertEquals(PIndexState.ACTIVE.toString(), rs.getString("INDEX_STATE"));
+ assertFalse(rs.next());
+
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
+ stmt.setString(1, "a");
+ stmt.setString(2, "x");
+ stmt.setString(3, "1");
+ stmt.execute();
+ conn.commit();
+
+ // find a RS which doesn't has CATALOG table
+ TableName catalogTable = TableName.valueOf("SYSTEM.CATALOG");
+ TableName indexTable = TableName.valueOf(fullIndexName);
+ final HBaseCluster cluster = this.util.getHBaseCluster();
+ Collection<ServerName> rss = cluster.getClusterStatus().getServers();
+ HBaseAdmin admin = this.util.getHBaseAdmin();
+ List<HRegionInfo> regions = admin.getTableRegions(catalogTable);
+ ServerName catalogRS = cluster.getServerHoldingRegion(regions.get(0).getTable(),
+ regions.get(0).getRegionName());
+ ServerName metaRS = cluster.getServerHoldingMeta();
+ ServerName rsToBeKilled = null;
+
+ // find first RS isn't holding META or CATALOG table
+ for(ServerName curRS : rss) {
+ if(!curRS.equals(catalogRS) && !metaRS.equals(curRS)) {
+ rsToBeKilled = curRS;
+ break;
+ }
+ }
+ assertTrue(rsToBeKilled != null);
+
+ regions = admin.getTableRegions(indexTable);
+ final HRegionInfo indexRegion = regions.get(0);
+ final ServerName dstRS = rsToBeKilled;
+ admin.move(indexRegion.getEncodedNameAsBytes(), Bytes.toBytes(rsToBeKilled.getServerName()));
+ this.util.waitFor(30000, 200, new Waiter.Predicate<Exception>() {
+ @Override
+ public boolean evaluate() throws Exception {
+ ServerName sn = cluster.getServerHoldingRegion(indexRegion.getTable(),
+ indexRegion.getRegionName());
+ return (sn != null && sn.equals(dstRS));
+ }
+ });
+
+ // use timer sending updates in every 10ms
+ this.scheduleTimer = new Timer(true);
+ this.scheduleTimer.schedule(new SendingUpdatesScheduleTask(conn, fullTableName), 0, 10);
+ // let timer sending some updates
+ Thread.sleep(100);
+
+ // kill RS hosting index table
+ this.util.getHBaseCluster().killRegionServer(rsToBeKilled);
+
+ // wait for index table completes recovery
+ this.util.waitUntilAllRegionsAssigned(indexTable);
+
+ // Verify the metadata for index is correct.
+ do {
+ Thread.sleep(15 * 1000); // sleep 15 secs
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
+ new String[] { PTableType.INDEX.toString() });
+ assertTrue(rs.next());
+ if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
+ break;
+ }
+ } while(true);
+ this.scheduleTimer.cancel();
+
+ assertEquals(cluster.getClusterStatus().getDeadServers(), 1);
+ }
}
static class SendingUpdatesScheduleTask extends TimerTask {
@@ -368,10 +442,12 @@ public class MutableIndexFailureIT extends BaseTest {
// running
private final static AtomicInteger inProgress = new AtomicInteger(0);
private final Connection conn;
+ private final String fullTableName;
private int inserts = 0;
- public SendingUpdatesScheduleTask(Connection conn) {
+ public SendingUpdatesScheduleTask(Connection conn, String fullTableName) {
this.conn = conn;
+ this.fullTableName = fullTableName;
}
public void run() {
@@ -382,7 +458,7 @@ public class MutableIndexFailureIT extends BaseTest {
try {
inProgress.incrementAndGet();
inserts++;
- PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)");
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
stmt.setString(1, "a" + inserts);
stmt.setString(2, "x" + inserts);
stmt.setString(3, String.valueOf(inserts));