You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ma...@apache.org on 2015/12/11 03:43:39 UTC
[01/52] [abbrv] phoenix git commit: PHOENIX-1674 Snapshot isolation
transaction support through Tephra (James Taylor, Thomas D'Silva)
Repository: phoenix
Updated Branches:
refs/heads/calcite 271de4d04 -> 5a833c326
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java
new file mode 100644
index 0000000..83d0b42
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema;
+
+public class PTableRef {
+ private final PTable table;
+ private final int estSize;
+ private volatile long lastAccessTime;
+ // timestamp (scn or txn timestamp at which rpc to fetch the table was made)
+ private long resolvedTimeStamp;
+
+ public PTableRef(PTable table, long lastAccessTime, int estSize, long resolvedTime) {
+ this.table = table;
+ this.lastAccessTime = lastAccessTime;
+ this.estSize = estSize;
+ this.resolvedTimeStamp = resolvedTime;
+ }
+
+ public PTableRef(PTable table, long lastAccessTime, long resolvedTime) {
+ this (table, lastAccessTime, table.getEstimatedSize(), resolvedTime);
+ }
+
+ public PTableRef(PTableRef tableRef) {
+ this (tableRef.table, tableRef.lastAccessTime, tableRef.estSize, tableRef.resolvedTimeStamp);
+ }
+
+ public PTable getTable() {
+ return table;
+ }
+
+ public long getResolvedTimeStamp() {
+ return resolvedTimeStamp;
+ }
+
+ public int getEstSize() {
+ return estSize;
+ }
+
+ public long getLastAccessTime() {
+ return lastAccessTime;
+ }
+
+ public void setLastAccessTime(long lastAccessTime) {
+ this.lastAccessTime = lastAccessTime;
+ }
+
+ public void setResolvedTimeStamp(long resolvedTimeStamp) {
+ this.resolvedTimeStamp = resolvedTimeStamp;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
index c429ac4..091b929 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
@@ -46,7 +46,10 @@ public enum TableProperty {
TTL(HColumnDescriptor.TTL, COLUMN_FAMILY_NOT_ALLOWED_FOR_TTL, true, CANNOT_ALTER_PROPERTY, false),
- STORE_NULLS(PhoenixDatabaseMetaData.STORE_NULLS, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, false);
+ STORE_NULLS(PhoenixDatabaseMetaData.STORE_NULLS, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, false),
+
+ TRANSACTIONAL(PhoenixDatabaseMetaData.TRANSACTIONAL, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, false),
+ ;
private final String propertyName;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
index 78b00fa..26cbbc3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
@@ -17,6 +17,8 @@
*/
package org.apache.phoenix.schema;
+import java.util.Objects;
+
import org.apache.hadoop.hbase.HConstants;
import org.apache.phoenix.compile.TupleProjectionCompiler;
import org.apache.phoenix.query.QueryConstants;
@@ -38,6 +40,10 @@ public class TableRef {
this(tableRef.alias, tableRef.table, timeStamp, tableRef.hasDynamicCols);
}
+ public TableRef(TableRef tableRef, String alias) {
+ this(alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.hasDynamicCols);
+ }
+
public TableRef(PTable table) {
this(null, table, HConstants.LATEST_TIMESTAMP, false);
}
@@ -101,7 +107,7 @@ public class TableRef {
public int hashCode() {
final int prime = 31;
int result = alias == null ? 0 : alias.hashCode();
- result = prime * result + this.table.getName().getString().hashCode();
+ result = prime * result + ( this.table.getName()!=null ? this.table.getName().hashCode() : 0);
return result;
}
@@ -111,8 +117,8 @@ public class TableRef {
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
TableRef other = (TableRef)obj;
- if ((alias == null && other.alias != null) || (alias != null && !alias.equals(other.alias))) return false;
- if (!table.getName().getString().equals(other.table.getName().getString())) return false;
+ if (!Objects.equals(alias, other.alias)) return false;
+ if (!Objects.equals(table.getName(), other.table.getName())) return false;
return true;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
index 159e0c9..c39df2c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricsRecord;
import org.apache.hadoop.metrics2.MetricsSink;
import org.apache.hadoop.metrics2.MetricsTag;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.metrics.MetricInfo;
import org.apache.phoenix.metrics.Metrics;
import org.apache.phoenix.query.QueryServices;
@@ -175,7 +176,13 @@ public class PhoenixMetricsSink implements MetricsSink {
TAG_COUNT + " smallint, " +
ANNOTATION_COUNT + " smallint" +
" CONSTRAINT pk PRIMARY KEY (" + TRACE.columnName + ", "
- + PARENT.columnName + ", " + SPAN.columnName + "))\n";
+ + PARENT.columnName + ", " + SPAN.columnName + "))\n" +
+ // We have a config parameter that can be set so that tables are
+ // transactional by default. If that's set, we still don't want these system
+ // tables created as transactional tables, make these table non
+ // transactional
+ PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
+;
PreparedStatement stmt = conn.prepareStatement(ddl);
stmt.execute();
this.table = table;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 3bf6f23..f361fb9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -30,6 +30,7 @@ import java.util.Map;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Mutation;
@@ -85,6 +86,8 @@ import org.apache.phoenix.schema.types.PDecimal;
import org.apache.phoenix.schema.types.PVarbinary;
import org.apache.phoenix.schema.types.PVarchar;
+import co.cask.tephra.TxConstants;
+
import com.google.common.collect.Lists;
public class IndexUtil {
@@ -209,13 +212,33 @@ public class IndexUtil {
.getLength()) == 0);
}
+ public static List<Delete> generateDeleteIndexData(final PTable table, PTable index,
+ List<Delete> dataMutations, ImmutableBytesWritable ptr, final KeyValueBuilder kvBuilder, PhoenixConnection connection)
+ throws SQLException {
+ try {
+ IndexMaintainer maintainer = index.getIndexMaintainer(table, connection);
+ List<Delete> indexMutations = Lists.newArrayListWithExpectedSize(dataMutations.size());
+ for (final Mutation dataMutation : dataMutations) {
+ long ts = MetaDataUtil.getClientTimeStamp(dataMutation);
+ ptr.set(dataMutation.getRow());
+ Delete delete = maintainer.buildDeleteMutation(kvBuilder, ptr, ts);
+ delete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, dataMutation.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY));
+ indexMutations.add(delete);
+ }
+ return indexMutations;
+ } catch (IOException e) {
+ throw new SQLException(e);
+ }
+ }
+
public static List<Mutation> generateIndexData(final PTable table, PTable index,
- List<Mutation> dataMutations, ImmutableBytesWritable ptr, final KeyValueBuilder kvBuilder, PhoenixConnection connection)
+ List<Mutation> dataMutations, final KeyValueBuilder kvBuilder, PhoenixConnection connection)
throws SQLException {
try {
+ final ImmutableBytesWritable ptr = new ImmutableBytesWritable();
IndexMaintainer maintainer = index.getIndexMaintainer(table, connection);
List<Mutation> indexMutations = Lists.newArrayListWithExpectedSize(dataMutations.size());
- for (final Mutation dataMutation : dataMutations) {
+ for (final Mutation dataMutation : dataMutations) {
long ts = MetaDataUtil.getClientTimeStamp(dataMutation);
ptr.set(dataMutation.getRow());
/*
@@ -235,7 +258,7 @@ public class IndexUtil {
}
@Override
- public ImmutableBytesPtr getLatestValue(ColumnReference ref) {
+ public ImmutableBytesWritable getLatestValue(ColumnReference ref) {
// Always return null for our empty key value, as this will cause the index
// maintainer to always treat this Put as a new row.
if (isEmptyKeyValue(table, ref)) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index ee73a58..47d3bc3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -272,6 +272,13 @@ public class MetaDataUtil {
return false;
}
+ public static boolean isTransactional(Mutation m, KeyValueBuilder builder, ImmutableBytesWritable ptr) {
+ if (getMutationValue(m, PhoenixDatabaseMetaData.TRANSACTIONAL_BYTES, builder, ptr)) {
+ return Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(ptr));
+ }
+ return false;
+ }
+
public static boolean isSalted(Mutation m, KeyValueBuilder builder, ImmutableBytesWritable ptr) {
return MetaDataUtil.getMutationValue(m, PhoenixDatabaseMetaData.SALT_BUCKETS_BYTES, builder, ptr);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
index 183bfa7..912dd03 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -511,7 +511,7 @@ public class ScanUtil {
}
}
- public static ScanRanges newScanRanges(List<Mutation> mutations) throws SQLException {
+ public static ScanRanges newScanRanges(List<? extends Mutation> mutations) throws SQLException {
List<KeyRange> keys = Lists.newArrayListWithExpectedSize(mutations.size());
for (Mutation m : mutations) {
keys.add(PVarbinary.INSTANCE.getKeyRange(m.getRow()));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
index 5414d4f..1693600 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
@@ -30,7 +30,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Properties;
+import java.util.Map;
import javax.annotation.Nullable;
@@ -51,13 +51,13 @@ import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PColumnFamily;
import org.apache.phoenix.schema.PDatum;
-import org.apache.phoenix.schema.PMetaData;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.RowKeySchema;
import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder;
import org.apache.phoenix.schema.SaltingUtil;
import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.TableProperty;
import org.apache.phoenix.schema.ValueSchema.Field;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PVarbinary;
@@ -262,6 +262,9 @@ public class SchemaUtil {
public static String getMetaDataEntityName(String schemaName, String tableName, String familyName, String columnName) {
if ((schemaName == null || schemaName.isEmpty()) && (tableName == null || tableName.isEmpty())) {
+ if (columnName == null || columnName.isEmpty()) {
+ return familyName;
+ }
return getName(familyName, columnName, false);
}
if ((familyName == null || familyName.isEmpty()) && (columnName == null || columnName.isEmpty())) {
@@ -390,6 +393,11 @@ public class SchemaUtil {
return families.isEmpty() ? table.getDefaultFamilyName() == null ? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES : table.getDefaultFamilyName().getBytes() : families.get(0).getName().getBytes();
}
+ public static String getEmptyColumnFamilyAsString(PTable table) {
+ List<PColumnFamily> families = table.getColumnFamilies();
+ return families.isEmpty() ? table.getDefaultFamilyName() == null ? QueryConstants.DEFAULT_COLUMN_FAMILY : table.getDefaultFamilyName().getString() : families.get(0).getName().getString();
+ }
+
public static ImmutableBytesPtr getEmptyColumnFamilyPtr(PTable table) {
List<PColumnFamily> families = table.getColumnFamilies();
return families.isEmpty() ? table.getDefaultFamilyName() == null ? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES_PTR : table.getDefaultFamilyName().getBytesPtr() : families.get(0)
@@ -517,15 +525,10 @@ public class SchemaUtil {
}
protected static PhoenixConnection addMetaDataColumn(PhoenixConnection conn, long scn, String columnDef) throws SQLException {
- String url = conn.getURL();
- Properties props = conn.getClientInfo();
- PMetaData metaData = conn.getMetaDataCache();
- props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(scn));
PhoenixConnection metaConnection = null;
-
Statement stmt = null;
try {
- metaConnection = new PhoenixConnection(conn.getQueryServices(), url, props, metaData);
+ metaConnection = new PhoenixConnection(conn.getQueryServices(), conn, scn);
try {
stmt = metaConnection.createStatement();
stmt.executeUpdate("ALTER TABLE SYSTEM.\"TABLE\" ADD IF NOT EXISTS " + columnDef);
@@ -711,6 +714,16 @@ public class SchemaUtil {
checkArgument(!isNullOrEmpty(columnName), "Column name cannot be null or empty");
return columnFamilyName == null ? ("\"" + columnName + "\"") : ("\"" + columnFamilyName + "\"" + QueryConstants.NAME_SEPARATOR + "\"" + columnName + "\"");
}
+
+ public static boolean hasHTableDescriptorProps(Map<String, Object> tableProps) {
+ int pTablePropCount = 0;
+ for (String prop : tableProps.keySet()) {
+ if (TableProperty.isPhoenixTableProperty(prop)) {
+ pTablePropCount++;
+ }
+ }
+ return tableProps.size() - pTablePropCount > 0;
+ }
/**
* Replaces all occurrences of {@link #ESCAPE_CHARACTER} with an empty character.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java
index a51dc84..679c7a1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.HTablePool;
+import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.phoenix.exception.PhoenixIOException;
import org.apache.phoenix.exception.SQLExceptionCode;
@@ -66,7 +67,9 @@ public class ServerUtil {
} else if (t instanceof IOException) {
// If the IOException does not wrap any exception, then bubble it up.
Throwable cause = t.getCause();
- if (cause == null || cause instanceof IOException) {
+ if (cause instanceof RetriesExhaustedWithDetailsException)
+ return new DoNotRetryIOException(t.getMessage(), cause);
+ else if (cause == null || cause instanceof IOException) {
return (IOException) t;
}
// Else assume it's been wrapped, so throw as DoNotRetryIOException to prevent client hanging while retrying
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java
new file mode 100644
index 0000000..9f16e52
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import java.sql.SQLException;
+
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.exception.SQLExceptionInfo;
+import org.apache.phoenix.execute.MutationState;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.schema.PTable;
+
+import co.cask.tephra.TransactionConflictException;
+import co.cask.tephra.TransactionFailureException;
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.hbase11.TransactionAwareHTable;
+
+public class TransactionUtil {
+ private TransactionUtil() {
+ }
+
+ public static long convertToNanoseconds(long serverTimeStamp) {
+ return serverTimeStamp * TxConstants.MAX_TX_PER_MS;
+ }
+
+ public static long convertToMilliseconds(long serverTimeStamp) {
+ return serverTimeStamp / TxConstants.MAX_TX_PER_MS;
+ }
+
+ public static SQLException getTransactionFailureException(TransactionFailureException e) {
+ if (e instanceof TransactionConflictException) {
+ return new SQLExceptionInfo.Builder(SQLExceptionCode.TRANSACTION_CONFLICT_EXCEPTION)
+ .setMessage(e.getMessage())
+ .setRootCause(e)
+ .build().buildException();
+
+ }
+ return new SQLExceptionInfo.Builder(SQLExceptionCode.TRANSACTION_FAILED)
+ .setMessage(e.getMessage())
+ .setRootCause(e)
+ .build().buildException();
+ }
+
+ public static TransactionAwareHTable getTransactionAwareHTable(HTableInterface htable, PTable table) {
+ // Conflict detection is not needed for tables with write-once/append-only data
+ return new TransactionAwareHTable(htable, table.isImmutableRows() ? TxConstants.ConflictDetection.NONE : TxConstants.ConflictDetection.ROW);
+ }
+
+ // we resolve transactional tables at the txn read pointer
+ public static long getResolvedTimestamp(PhoenixConnection connection, boolean isTransactional, long defaultResolvedTimestamp) {
+ MutationState mutationState = connection.getMutationState();
+ Long scn = connection.getSCN();
+ return scn != null ? scn : (isTransactional && mutationState.isTransactionStarted()) ? convertToMilliseconds(mutationState.getReadPointer()) : defaultResolvedTimestamp;
+ }
+
+ public static long getResolvedTime(PhoenixConnection connection, MetaDataMutationResult result) {
+ PTable table = result.getTable();
+ boolean isTransactional = table!=null && table.isTransactional();
+ return getResolvedTimestamp(connection, isTransactional, result.getMutationTime());
+ }
+
+ public static long getResolvedTimestamp(PhoenixConnection connection, MetaDataMutationResult result) {
+ PTable table = result.getTable();
+ MutationState mutationState = connection.getMutationState();
+ boolean txInProgress = table != null && table.isTransactional() && mutationState.isTransactionStarted();
+ return txInProgress ? convertToMilliseconds(mutationState.getReadPointer()) : result.getMutationTime();
+ }
+
+ public static Long getTableTimestamp(PhoenixConnection connection, boolean transactional) throws SQLException {
+ Long timestamp = null;
+ if (!transactional) {
+ return timestamp;
+ }
+ MutationState mutationState = connection.getMutationState();
+ // we need to burn a txn so that we are sure the txn read pointer is close to wall clock time
+ if (!mutationState.isTransactionStarted()) {
+ mutationState.startTransaction();
+ connection.commit();
+ }
+ else {
+ connection.commit();
+ }
+ mutationState.startTransaction();
+ timestamp = convertToMilliseconds(mutationState.getReadPointer());
+ connection.commit();
+ return timestamp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
index 23eb147..9da1a67 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
@@ -1482,7 +1482,7 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest {
statement.execute("create local index my_idx on example (fn) DEFAULT_COLUMN_FAMILY='F'");
fail();
} catch (SQLException e) {
- assertEquals(SQLExceptionCode.VIEW_WITH_PROPERTIES.getErrorCode(),e.getErrorCode());
+ assertEquals(SQLExceptionCode.DEFAULT_COLUMN_FAMILY_ON_SHARED_TABLE.getErrorCode(),e.getErrorCode());
}
statement.execute("create local index my_idx on example (fn)");
} finally {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/compile/ViewCompilerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/ViewCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/ViewCompilerTest.java
index 1d95058..959031f 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/ViewCompilerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/ViewCompilerTest.java
@@ -31,20 +31,20 @@ import org.apache.phoenix.query.BaseConnectionlessQueryTest;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.ViewType;
-import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.util.PropertiesUtil;
import org.junit.Test;
public class ViewCompilerTest extends BaseConnectionlessQueryTest {
@Test
public void testViewTypeCalculation() throws Exception {
- assertViewType(new String[] {
+ assertViewType(new String[] {"V1","V2","V3","V4"}, new String[] {
"CREATE VIEW v1 AS SELECT * FROM t WHERE k1 = 1 AND k2 = 'foo'",
"CREATE VIEW v2 AS SELECT * FROM t WHERE k2 = 'foo'",
"CREATE VIEW v3 AS SELECT * FROM t WHERE v = 'bar'||'bas'",
"CREATE VIEW v4 AS SELECT * FROM t WHERE 'bar'=v and 5+3/2 = k1",
}, ViewType.UPDATABLE);
- assertViewType(new String[] {
+ assertViewType(new String[] {"V1","V2","V3","V4"}, new String[] {
"CREATE VIEW v1 AS SELECT * FROM t WHERE k1 < 1 AND k2 = 'foo'",
"CREATE VIEW v2 AS SELECT * FROM t WHERE substr(k2,0,3) = 'foo'",
"CREATE VIEW v3 AS SELECT * FROM t WHERE v = TO_CHAR(CURRENT_DATE())",
@@ -52,28 +52,27 @@ public class ViewCompilerTest extends BaseConnectionlessQueryTest {
}, ViewType.READ_ONLY);
}
- public void assertViewType(String[] views, ViewType viewType) throws Exception {
+ public void assertViewType(String[] viewNames, String[] viewDDLs, ViewType viewType) throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
PhoenixConnection conn = DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class);
String ct = "CREATE TABLE t (k1 INTEGER NOT NULL, k2 VARCHAR, v VARCHAR, CONSTRAINT pk PRIMARY KEY (k1,k2))";
conn.createStatement().execute(ct);
- for (String view : views) {
- conn.createStatement().execute(view);
+ for (String viewDDL : viewDDLs) {
+ conn.createStatement().execute(viewDDL);
}
StringBuilder buf = new StringBuilder();
int count = 0;
- for (PTable table : conn.getMetaDataCache()) {
- if (table.getType() == PTableType.VIEW) {
- assertEquals(viewType, table.getViewType());
- conn.createStatement().execute("DROP VIEW " + table.getName().getString());
- buf.append(' ');
- buf.append(table.getName().getString());
- count++;
- }
+ for (String view : viewNames) {
+ PTable table = conn.getTable(new PTableKey(null, view));
+ assertEquals(viewType, table.getViewType());
+ conn.createStatement().execute("DROP VIEW " + table.getName().getString());
+ buf.append(' ');
+ buf.append(table.getName().getString());
+ count++;
}
- assertEquals("Expected " + views.length + ", but got " + count + ":"+ buf.toString(), views.length, count);
+ assertEquals("Expected " + viewDDLs.length + ", but got " + count + ":"+ buf.toString(), viewDDLs.length, count);
}
@Test
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
index 334ce8c..9d5e17c 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
@@ -233,7 +233,7 @@ public class CorrelatePlanTest {
PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM,
null, null, columns, null, null, Collections.<PTable>emptyList(),
false, Collections.<PName>emptyList(), null, null, false, false, false, null,
- null, null, true);
+ null, null, true, false);
TableRef sourceTable = new TableRef(pTable);
List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList();
for (PColumn column : sourceTable.getTable().getColumns()) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
index d142672..a0aca4d 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.SortedMap;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.compile.QueryPlan;
import org.apache.phoenix.end2end.Shadower;
@@ -670,7 +671,7 @@ public class SkipScanBigFilterTest extends BaseConnectionlessQueryTest {
return table.getTimeStamp()+1;
}
});
- conn.unwrap(PhoenixConnection.class).addTable(tableWithStats);
+ conn.unwrap(PhoenixConnection.class).addTable(tableWithStats, System.currentTimeMillis());
String query = "SELECT count(1) cnt,\n" +
" coalesce(SUM(impressions), 0.0) AS \"impressions\",\n" +
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/CoveredIndexCodecForTesting.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/CoveredIndexCodecForTesting.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/CoveredIndexCodecForTesting.java
index b0fae03..7d31516 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/CoveredIndexCodecForTesting.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/covered/CoveredIndexCodecForTesting.java
@@ -25,49 +25,46 @@ import java.util.List;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.phoenix.hbase.index.builder.BaseIndexCodec;
-import org.apache.phoenix.hbase.index.covered.IndexCodec;
-import org.apache.phoenix.hbase.index.covered.IndexUpdate;
-import org.apache.phoenix.hbase.index.covered.TableState;
/**
- * An {@link IndexCodec} for testing that allow you to specify the index updates/deletes, regardless
- * of the current tables' state.
+ * An {@link IndexCodec} for testing that allow you to specify the index updates/deletes, regardless of the current
+ * tables' state.
*/
public class CoveredIndexCodecForTesting extends BaseIndexCodec {
- private List<IndexUpdate> deletes = new ArrayList<IndexUpdate>();
- private List<IndexUpdate> updates = new ArrayList<IndexUpdate>();
+ private List<IndexUpdate> deletes = new ArrayList<IndexUpdate>();
+ private List<IndexUpdate> updates = new ArrayList<IndexUpdate>();
- public void addIndexDelete(IndexUpdate... deletes) {
- this.deletes.addAll(Arrays.asList(deletes));
- }
-
- public void addIndexUpserts(IndexUpdate... updates) {
- this.updates.addAll(Arrays.asList(updates));
- }
+ public void addIndexDelete(IndexUpdate... deletes) {
+ this.deletes.addAll(Arrays.asList(deletes));
+ }
- public void clear() {
- this.deletes.clear();
- this.updates.clear();
- }
-
- @Override
- public Iterable<IndexUpdate> getIndexDeletes(TableState state, IndexMetaData context) {
- return this.deletes;
- }
+ public void addIndexUpserts(IndexUpdate... updates) {
+ this.updates.addAll(Arrays.asList(updates));
+ }
- @Override
- public Iterable<IndexUpdate> getIndexUpserts(TableState state, IndexMetaData context) {
- return this.updates;
- }
+ public void clear() {
+ this.deletes.clear();
+ this.updates.clear();
+ }
- @Override
- public void initialize(RegionCoprocessorEnvironment env) throws IOException {
- // noop
- }
+ @Override
+ public Iterable<IndexUpdate> getIndexDeletes(TableState state, IndexMetaData context) {
+ return this.deletes;
+ }
- @Override
- public boolean isEnabled(Mutation m) {
- return true;
- }
+ @Override
+ public Iterable<IndexUpdate> getIndexUpserts(TableState state, IndexMetaData context) {
+ return this.updates;
+ }
+
+ @Override
+ public void initialize(RegionCoprocessorEnvironment env) throws IOException {
+ // noop
+ }
+
+ @Override
+ public boolean isEnabled(Mutation m) {
+ return true;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java
index 27ed370..e941ceb 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java
@@ -73,7 +73,7 @@ public class IndexMaintainerTest extends BaseConnectionlessQueryTest {
return new ValueGetter() {
@Override
- public ImmutableBytesPtr getLatestValue(ColumnReference ref) {
+ public ImmutableBytesWritable getLatestValue(ColumnReference ref) {
return new ImmutableBytesPtr(valueMap.get(ref));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
index d4956ee..f6df5c1 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
@@ -88,9 +88,9 @@ public class PhoenixTestDriver extends PhoenixEmbeddedDriver {
if (connectionQueryServices != null) { return connectionQueryServices; }
ConnectionInfo connInfo = ConnectionInfo.create(url);
if (connInfo.isConnectionless()) {
- connectionQueryServices = new ConnectionlessQueryServicesImpl(queryServices, connInfo);
+ connectionQueryServices = new ConnectionlessQueryServicesImpl(queryServices, connInfo, info);
} else {
- connectionQueryServices = new ConnectionQueryServicesTestImpl(queryServices, connInfo);
+ connectionQueryServices = new ConnectionQueryServicesTestImpl(queryServices, connInfo, info);
}
connectionQueryServices.init(url, info);
return connectionQueryServices;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index 8ad4f12..fa5df31 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -82,11 +82,13 @@ import static org.apache.phoenix.util.TestUtil.STABLE_NAME;
import static org.apache.phoenix.util.TestUtil.TABLE_WITH_ARRAY;
import static org.apache.phoenix.util.TestUtil.TABLE_WITH_SALTING;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.apache.phoenix.util.TestUtil.TRANSACTIONAL_DATA_TABLE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
@@ -106,6 +108,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
@@ -130,6 +133,7 @@ import org.apache.phoenix.end2end.BaseClientManagedTimeIT;
import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver;
+import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
import org.apache.phoenix.jdbc.PhoenixTestDriver;
import org.apache.phoenix.schema.NewerTableAlreadyExistsException;
import org.apache.phoenix.schema.PTableType;
@@ -143,12 +147,28 @@ import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
+import org.apache.twill.discovery.DiscoveryService;
+import org.apache.twill.discovery.ZKDiscoveryService;
+import org.apache.twill.internal.utils.Networks;
+import org.apache.twill.zookeeper.RetryStrategies;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.apache.twill.zookeeper.ZKClientServices;
+import org.apache.twill.zookeeper.ZKClients;
+import org.junit.ClassRule;
+import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import co.cask.tephra.TransactionManager;
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.distributed.TransactionService;
+import co.cask.tephra.metrics.TxMetricsCollector;
+import co.cask.tephra.persist.InMemoryTransactionStateStorage;
+
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import com.google.inject.util.Providers;
/**
*
@@ -187,7 +207,13 @@ public abstract class BaseTest {
" CONSTRAINT pk PRIMARY KEY (varchar_pk, char_pk, int_pk, long_pk DESC, decimal_pk, date_pk)) ";
private static final Map<String,String> tableDDLMap;
private static final Logger logger = LoggerFactory.getLogger(BaseTest.class);
-
+ protected static final int DEFAULT_TXN_TIMEOUT_SECONDS = 30;
+ private static ZKClientService zkClient;
+ private static TransactionService txService;
+ protected static TransactionManager txManager;
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+
static {
ImmutableMap.Builder<String,String> builder = ImmutableMap.builder();
builder.put(ENTITY_HISTORY_TABLE_NAME,"create table " + ENTITY_HISTORY_TABLE_NAME +
@@ -401,48 +427,9 @@ public abstract class BaseTest {
builder.put("KVBigIntValueTest", "create table KVBigIntValueTest" +
" (pk integer not null primary key,\n" +
" kv bigint)\n");
- builder.put(INDEX_DATA_TABLE, "create table " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + INDEX_DATA_TABLE + "(" +
- " varchar_pk VARCHAR NOT NULL, " +
- " char_pk CHAR(6) NOT NULL, " +
- " int_pk INTEGER NOT NULL, "+
- " long_pk BIGINT NOT NULL, " +
- " decimal_pk DECIMAL(31, 10) NOT NULL, " +
- " date_pk DATE NOT NULL, " +
- " a.varchar_col1 VARCHAR, " +
- " a.char_col1 CHAR(10), " +
- " a.int_col1 INTEGER, " +
- " a.long_col1 BIGINT, " +
- " a.decimal_col1 DECIMAL(31, 10), " +
- " a.date1 DATE, " +
- " b.varchar_col2 VARCHAR, " +
- " b.char_col2 CHAR(10), " +
- " b.int_col2 INTEGER, " +
- " b.long_col2 BIGINT, " +
- " b.decimal_col2 DECIMAL(31, 10), " +
- " b.date2 DATE " +
- " CONSTRAINT pk PRIMARY KEY (varchar_pk, char_pk, int_pk, long_pk DESC, decimal_pk, date_pk)) " +
- "IMMUTABLE_ROWS=true");
- builder.put(MUTABLE_INDEX_DATA_TABLE, "create table " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE + "(" +
- " varchar_pk VARCHAR NOT NULL, " +
- " char_pk CHAR(6) NOT NULL, " +
- " int_pk INTEGER NOT NULL, "+
- " long_pk BIGINT NOT NULL, " +
- " decimal_pk DECIMAL(31, 10) NOT NULL, " +
- " date_pk DATE NOT NULL, " +
- " a.varchar_col1 VARCHAR, " +
- " a.char_col1 CHAR(10), " +
- " a.int_col1 INTEGER, " +
- " a.long_col1 BIGINT, " +
- " a.decimal_col1 DECIMAL(31, 10), " +
- " a.date1 DATE, " +
- " b.varchar_col2 VARCHAR, " +
- " b.char_col2 CHAR(10), " +
- " b.int_col2 INTEGER, " +
- " b.long_col2 BIGINT, " +
- " b.decimal_col2 DECIMAL(31, 10), " +
- " b.date2 DATE " +
- " CONSTRAINT pk PRIMARY KEY (varchar_pk, char_pk, int_pk, long_pk DESC, decimal_pk, date_pk)) "
- );
+ builder.put(INDEX_DATA_TABLE, "create table " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + INDEX_DATA_TABLE + TEST_TABLE_SCHEMA + "IMMUTABLE_ROWS=true");
+ builder.put(MUTABLE_INDEX_DATA_TABLE, "create table " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE + TEST_TABLE_SCHEMA);
+ builder.put(TRANSACTIONAL_DATA_TABLE, "create table " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + TRANSACTIONAL_DATA_TABLE + TEST_TABLE_SCHEMA + "TRANSACTIONAL=true");
builder.put("SumDoubleTest","create table SumDoubleTest" +
" (id varchar not null primary key, d DOUBLE, f FLOAT, ud UNSIGNED_DOUBLE, uf UNSIGNED_FLOAT, i integer, de decimal)");
builder.put(JOIN_ORDER_TABLE_FULL_NAME, "create table " + JOIN_ORDER_TABLE_FULL_NAME +
@@ -499,9 +486,9 @@ public abstract class BaseTest {
return conf.get(QueryServices.ZOOKEEPER_PORT_ATTRIB);
}
- private static String url;
+ protected static String url;
protected static PhoenixTestDriver driver;
- private static boolean clusterInitialized = false;
+ protected static boolean clusterInitialized = false;
private static HBaseTestingUtility utility;
protected static final Configuration config = HBaseConfiguration.create();
@@ -512,10 +499,53 @@ public abstract class BaseTest {
return url;
}
- protected static String checkClusterInitialized(ReadOnlyProps overrideProps) {
+ private static void teardownTxManager() throws SQLException {
+ try {
+ if (txService != null) txService.stopAndWait();
+ } finally {
+ try {
+ if (zkClient != null) zkClient.stopAndWait();
+ } finally {
+ txService = null;
+ zkClient = null;
+ }
+ }
+
+ }
+
+ protected static void setupTxManager() throws SQLException, IOException {
+ config.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
+ config.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times");
+ config.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1);
+ config.setInt(TxConstants.Service.CFG_DATA_TX_BIND_PORT, Networks.getRandomPort());
+ config.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());
+ config.setInt(TxConstants.Manager.CFG_TX_TIMEOUT, DEFAULT_TXN_TIMEOUT_SECONDS);
+
+ ConnectionInfo connInfo = ConnectionInfo.create(getUrl());
+ zkClient = ZKClientServices.delegate(
+ ZKClients.reWatchOnExpire(
+ ZKClients.retryOnFailure(
+ ZKClientService.Builder.of(connInfo.getZookeeperConnectionString())
+ .setSessionTimeout(config.getInt(HConstants.ZK_SESSION_TIMEOUT,
+ HConstants.DEFAULT_ZK_SESSION_TIMEOUT))
+ .build(),
+ RetryStrategies.exponentialDelay(500, 2000, TimeUnit.MILLISECONDS)
+ )
+ )
+ );
+ zkClient.startAndWait();
+
+ DiscoveryService discovery = new ZKDiscoveryService(zkClient);
+ txManager = new TransactionManager(config, new InMemoryTransactionStateStorage(), new TxMetricsCollector());
+ txService = new TransactionService(config, zkClient, discovery, Providers.of(txManager));
+ txService.startAndWait();
+ }
+
+ protected static String checkClusterInitialized(ReadOnlyProps overrideProps) throws Exception {
if (!clusterInitialized) {
url = setUpTestCluster(config, overrideProps);
clusterInitialized = true;
+ setupTxManager();
}
return url;
}
@@ -560,8 +590,12 @@ public abstract class BaseTest {
utility.shutdownMiniCluster();
}
} finally {
- utility = null;
- clusterInitialized = false;
+ try {
+ teardownTxManager();
+ } finally {
+ utility = null;
+ clusterInitialized = false;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
index 732563f..089c5f1 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
@@ -20,7 +20,6 @@ package org.apache.phoenix.query;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.sql.Connection;
@@ -150,7 +149,7 @@ public class ConnectionlessTest {
assertTrue(iterator.hasNext());
kv = iterator.next();
assertArrayEquals(expectedRowKey1, kv.getRow());
- assertNull(PVarchar.INSTANCE.toObject(kv.getValue()));
+ assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, PVarchar.INSTANCE.toObject(kv.getValue()));
}
private static void assertRow2(Iterator<KeyValue> iterator, byte[] expectedRowKey2) {
@@ -166,7 +165,7 @@ public class ConnectionlessTest {
assertTrue(iterator.hasNext());
kv = iterator.next();
assertArrayEquals(expectedRowKey2, kv.getRow());
- assertNull(PVarchar.INSTANCE.toObject(kv.getValue()));
+ assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, PVarchar.INSTANCE.toObject(kv.getValue()));
}
@Test
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java
index e4ac5d3..14b5f1f 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java
@@ -31,7 +31,7 @@ public class PMetaDataImplTest {
private static PMetaData addToTable(PMetaData metaData, String name, int size) throws SQLException {
PTable table = new PSizedTable(new PTableKey(null,name), size);
- return metaData.addTable(table);
+ return metaData.addTable(table, System.currentTimeMillis());
}
private static PMetaData removeFromTable(PMetaData metaData, String name) throws SQLException {
@@ -39,7 +39,7 @@ public class PMetaDataImplTest {
}
private static PTable getFromTable(PMetaData metaData, String name) throws TableNotFoundException {
- return metaData.getTable(new PTableKey(null,name));
+ return metaData.getTableRef(new PTableKey(null,name)).getTable();
}
private static void assertNames(PMetaData metaData, String... names) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index 0fb9473..451ce62 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -25,14 +25,17 @@ import static org.apache.phoenix.util.PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PAR
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.sql.Connection;
+import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
+import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
@@ -98,7 +101,7 @@ public class TestUtil {
public static final String DEFAULT_INDEX_TABLE_NAME = "I";
public static final String DEFAULT_DATA_TABLE_FULL_NAME = SchemaUtil.getTableName(DEFAULT_SCHEMA_NAME, "T");
public static final String DEFAULT_INDEX_TABLE_FULL_NAME = SchemaUtil.getTableName(DEFAULT_SCHEMA_NAME, "I");
-
+
private TestUtil() {
}
@@ -191,6 +194,7 @@ public class TestUtil {
public static final String INDEX_DATA_SCHEMA = "INDEX_TEST";
public static final String INDEX_DATA_TABLE = "INDEX_DATA_TABLE";
public static final String MUTABLE_INDEX_DATA_TABLE = "MUTABLE_INDEX_DATA_TABLE";
+ public static final String TRANSACTIONAL_DATA_TABLE = "TRANSACTIONAL_DATA_TABLE";
public static final String JOIN_SCHEMA = "Join";
public static final String JOIN_ORDER_TABLE = "OrderTable";
public static final String JOIN_CUSTOMER_TABLE = "CustomerTable";
@@ -582,4 +586,36 @@ public class TestUtil {
analyzeTable(conn, tableName);
conn.close();
}
+
+ public static void setRowKeyColumns(PreparedStatement stmt, int i) throws SQLException {
+ // insert row
+ stmt.setString(1, "varchar" + String.valueOf(i));
+ stmt.setString(2, "char" + String.valueOf(i));
+ stmt.setInt(3, i);
+ stmt.setLong(4, i);
+ stmt.setBigDecimal(5, new BigDecimal(i*0.5d));
+ Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * TestUtil.NUM_MILLIS_IN_DAY);
+ stmt.setDate(6, date);
+ }
+
+ public static void validateRowKeyColumns(ResultSet rs, int i) throws SQLException {
+ assertTrue(rs.next());
+ assertEquals(rs.getString(1), "varchar" + String.valueOf(i));
+ assertEquals(rs.getString(2), "char" + String.valueOf(i));
+ assertEquals(rs.getInt(3), i);
+ assertEquals(rs.getInt(4), i);
+ assertEquals(rs.getBigDecimal(5), new BigDecimal(i*0.5d));
+ Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * TestUtil.NUM_MILLIS_IN_DAY);
+ assertEquals(rs.getDate(6), date);
+ }
+
+ public static String getTableName(Boolean mutable, Boolean transactional) {
+ StringBuilder tableNameBuilder = new StringBuilder(DEFAULT_DATA_TABLE_NAME);
+ if (mutable!=null)
+ tableNameBuilder.append(mutable ? "_MUTABLE" : "_IMMUTABLE");
+ if (transactional!=null)
+ tableNameBuilder.append(transactional ? "_TXN" : "_NON_TXN");
+ return tableNameBuilder.toString();
+ }
}
+
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-flume/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-flume/pom.xml b/phoenix-flume/pom.xml
index 1ab3fe7..b371567 100644
--- a/phoenix-flume/pom.xml
+++ b/phoenix-flume/pom.xml
@@ -164,6 +164,13 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
</dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <type>test-jar</type>
+ <version>${tephra.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-pig/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-pig/pom.xml b/phoenix-pig/pom.xml
index 7eef250..53406ff 100644
--- a/phoenix-pig/pom.xml
+++ b/phoenix-pig/pom.xml
@@ -160,6 +160,13 @@
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <type>test-jar</type>
+ <version>${tephra.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-protocol/src/main/PTable.proto
----------------------------------------------------------------------
diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto
index 3257a70..3048a40 100644
--- a/phoenix-protocol/src/main/PTable.proto
+++ b/phoenix-protocol/src/main/PTable.proto
@@ -86,4 +86,5 @@ message PTable {
optional bool storeNulls = 24;
optional int32 baseColumnCount = 25;
optional bool rowKeyOrderOptimizable = 26;
+ optional bool transactional = 27;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-protocol/src/main/ServerCachingService.proto
----------------------------------------------------------------------
diff --git a/phoenix-protocol/src/main/ServerCachingService.proto b/phoenix-protocol/src/main/ServerCachingService.proto
index 63aeef9..dcbaabd 100644
--- a/phoenix-protocol/src/main/ServerCachingService.proto
+++ b/phoenix-protocol/src/main/ServerCachingService.proto
@@ -37,6 +37,7 @@ message AddServerCacheRequest {
required bytes cacheId = 2;
required ImmutableBytesWritable cachePtr = 3;
required ServerCacheFactory cacheFactory = 4;
+ optional bytes txState = 5;
}
message AddServerCacheResponse {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-server/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-server/pom.xml b/phoenix-server/pom.xml
index 89ea5b5..2584748 100644
--- a/phoenix-server/pom.xml
+++ b/phoenix-server/pom.xml
@@ -106,5 +106,12 @@
<artifactId>hbase-testing-util</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <type>test-jar</type>
+ <version>${tephra.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3eb1f7c..148bb51 100644
--- a/pom.xml
+++ b/pom.xml
@@ -98,7 +98,7 @@
<commons-logging.version>1.2</commons-logging.version>
<commons-csv.version>1.0</commons-csv.version>
<sqlline.version>1.1.8</sqlline.version>
- <guava.version>12.0.1</guava.version>
+ <guava.version>13.0.1</guava.version>
<jackson.version>1.8.8</jackson.version>
<flume.version>1.4.0</flume.version>
<findbugs.version>1.3.2</findbugs.version>
@@ -112,6 +112,7 @@
<joni.version>2.1.2</joni.version>
<calcite.version>1.5.0</calcite.version>
<jettyVersion>8.1.7.v20120910</jettyVersion>
+ <tephra.version>0.6.3</tephra.version>
<!-- Test Dependencies -->
<mockito-all.version>1.8.5</mockito-all.version>
@@ -126,7 +127,7 @@
<maven-dependency-plugin.version>2.1</maven-dependency-plugin.version>
<maven.assembly.version>2.5.2</maven.assembly.version>
<maven.rat.version>0.8</maven.rat.version>
-
+
<!-- Plugin options -->
<numForkedUT>3</numForkedUT>
<numForkedIT>5</numForkedIT>
[26/52] [abbrv] phoenix git commit: PHOENIX-2461 Reverting Tweak
shutdown of ConnectionQueryService during testing
Posted by ma...@apache.org.
PHOENIX-2461 Reverting Tweak shutdown of ConnectionQueryService during testing
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/bf7187c6
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/bf7187c6
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/bf7187c6
Branch: refs/heads/calcite
Commit: bf7187c63c84bdb7d59c47f5625a94a14322a133
Parents: 3eb17d4
Author: James Taylor <jt...@salesforce.com>
Authored: Mon Nov 30 11:58:21 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Nov 30 11:58:21 2015 -0800
----------------------------------------------------------------------
.../phoenix/end2end/ConnectionQueryServicesTestImpl.java | 9 +--------
.../apache/phoenix/query/ConnectionQueryServicesImpl.java | 3 +--
2 files changed, 2 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/bf7187c6/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
index d3b3f8a..2028223 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
@@ -63,14 +63,7 @@ public class ConnectionQueryServicesTestImpl extends ConnectionQueryServicesImpl
try {
clearCache();
} finally {
- try {
- // Since ConnectionQueryServicesTestImpl is a singleton
- // during testing, attempt to stop the Zookeeper client
- // service when the driver is closed.
- // if (zkClientService != null) zkClientService.stopAndWait();
- } finally {
- super.close();
- }
+ super.close();
}
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/bf7187c6/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 7e1aee0..17df42d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -220,7 +220,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private HConnection connection;
private TransactionServiceClient txServiceClient;
- protected volatile ZKClientService zkClientService;
private volatile boolean initialized;
private volatile int nSequenceSaltBuckets;
@@ -306,7 +305,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private void initTxServiceClient() {
String zkQuorumServersString = connectionInfo.getZookeeperQuorum()+":"+connectionInfo.getPort();
- zkClientService = ZKClientServices.delegate(
+ ZKClientService zkClientService = ZKClientServices.delegate(
ZKClients.reWatchOnExpire(
ZKClients.retryOnFailure(
ZKClientService.Builder.of(zkQuorumServersString)
[46/52] [abbrv] phoenix git commit: PHOENIX-2481 JSON bulkload tool
Posted by ma...@apache.org.
PHOENIX-2481 JSON bulkload tool
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/578979a1
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/578979a1
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/578979a1
Branch: refs/heads/calcite
Commit: 578979a1437124d53e319ad554b72793bcef6fd3
Parents: 8ce3b58
Author: Nick Dimiduk <nd...@apache.org>
Authored: Mon Nov 16 17:18:34 2015 -0800
Committer: Nick Dimiduk <nd...@apache.org>
Committed: Mon Dec 7 13:09:09 2015 -0800
----------------------------------------------------------------------
.../phoenix/mapreduce/CsvBulkLoadToolIT.java | 16 +-
.../phoenix/mapreduce/AbstractBulkLoadTool.java | 402 +++++++++++++++
.../phoenix/mapreduce/CsvBulkImportUtil.java | 20 +-
.../phoenix/mapreduce/CsvBulkLoadTool.java | 514 +------------------
.../phoenix/mapreduce/CsvToKeyValueMapper.java | 226 +-------
.../phoenix/mapreduce/CsvToKeyValueReducer.java | 55 --
.../mapreduce/FormatToKeyValueMapper.java | 259 ++++++++++
.../mapreduce/FormatToKeyValueReducer.java | 54 ++
.../ImportPreUpsertKeyValueProcessor.java | 3 +-
.../phoenix/mapreduce/JsonBulkLoadTool.java | 53 ++
.../phoenix/mapreduce/JsonToKeyValueMapper.java | 75 +++
.../mapreduce/MultiHfileOutputFormat.java | 38 +-
.../mapreduce/bulkload/CsvTableRowkeyPair.java | 139 -----
.../mapreduce/bulkload/TableRowkeyPair.java | 134 +++++
.../mapreduce/bulkload/TargetTableRef.java | 70 +++
.../bulkload/TargetTableRefFunctions.java | 95 ++++
.../util/PhoenixConfigurationUtil.java | 15 +-
.../apache/phoenix/util/CSVCommonsLoader.java | 160 +-----
.../org/apache/phoenix/util/SchemaUtil.java | 145 +++++-
.../org/apache/phoenix/util/UpsertExecutor.java | 156 ++++++
.../phoenix/util/csv/CsvUpsertExecutor.java | 131 +----
.../phoenix/util/json/JsonUpsertExecutor.java | 209 ++++++++
.../util/json/ObjectToArrayConverter.java | 69 +++
.../phoenix/mapreduce/BulkLoadToolTest.java | 78 +++
.../mapreduce/CsvBulkImportUtilTest.java | 18 +-
.../phoenix/mapreduce/CsvBulkLoadToolTest.java | 69 ---
.../mapreduce/CsvToKeyValueMapperTest.java | 84 +--
.../mapreduce/FormatToKeyValueMapperTest.java | 102 ++++
.../util/AbstractUpsertExecutorTest.java | 136 +++++
.../phoenix/util/csv/CsvUpsertExecutorTest.java | 144 ++----
.../util/json/JsonUpsertExecutorTest.java | 53 ++
31 files changed, 2222 insertions(+), 1500 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
index 0e74d7b..a5b7488 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
@@ -17,14 +17,6 @@
*/
package org.apache.phoenix.mapreduce;
-import static org.apache.phoenix.query.BaseTest.setUpConfigForMiniCluster;
-import static org.apache.phoenix.query.QueryServices.DATE_FORMAT_ATTRIB;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
import java.io.PrintWriter;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -47,6 +39,14 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import static org.apache.phoenix.query.BaseTest.setUpConfigForMiniCluster;
+import static org.apache.phoenix.query.QueryServices.DATE_FORMAT_ATTRIB;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
@Category(NeedsOwnMiniClusterTest.class)
public class CsvBulkLoadToolIT {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
new file mode 100644
index 0000000..cf9ddef
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.jdbc.PhoenixDriver;
+import org.apache.phoenix.mapreduce.bulkload.TableRowkeyPair;
+import org.apache.phoenix.mapreduce.bulkload.TargetTableRef;
+import org.apache.phoenix.mapreduce.bulkload.TargetTableRefFunctions;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.util.ColumnInfo;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.StringUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Base tool for running MapReduce-based ingests of data.
+ */
+public abstract class AbstractBulkLoadTool extends Configured implements Tool {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(AbstractBulkLoadTool.class);
+
+ static final Option ZK_QUORUM_OPT = new Option("z", "zookeeper", true, "Supply zookeeper connection details (optional)");
+ static final Option INPUT_PATH_OPT = new Option("i", "input", true, "Input path (mandatory)");
+ static final Option OUTPUT_PATH_OPT = new Option("o", "output", true, "Output path for temporary HFiles (optional)");
+ static final Option SCHEMA_NAME_OPT = new Option("s", "schema", true, "Phoenix schema name (optional)");
+ static final Option TABLE_NAME_OPT = new Option("t", "table", true, "Phoenix table name (mandatory)");
+ static final Option INDEX_TABLE_NAME_OPT = new Option("it", "index-table", true, "Phoenix index table name when just loading this particualar index table");
+ static final Option IMPORT_COLUMNS_OPT = new Option("c", "import-columns", true, "Comma-separated list of columns to be imported");
+ static final Option IGNORE_ERRORS_OPT = new Option("g", "ignore-errors", false, "Ignore input errors");
+ static final Option HELP_OPT = new Option("h", "help", false, "Show this help and quit");
+
+ /**
+ * Set configuration values based on parsed command line options.
+ *
+ * @param cmdLine supplied command line options
+ * @param importColumns descriptors of columns to be imported
+ * @param conf job configuration
+ */
+ protected abstract void configureOptions(CommandLine cmdLine, List<ColumnInfo> importColumns,
+ Configuration conf) throws SQLException;
+ protected abstract void setupJob(Job job);
+
+ protected Options getOptions() {
+ Options options = new Options();
+ options.addOption(INPUT_PATH_OPT);
+ options.addOption(TABLE_NAME_OPT);
+ options.addOption(INDEX_TABLE_NAME_OPT);
+ options.addOption(ZK_QUORUM_OPT);
+ options.addOption(OUTPUT_PATH_OPT);
+ options.addOption(SCHEMA_NAME_OPT);
+ options.addOption(IMPORT_COLUMNS_OPT);
+ options.addOption(IGNORE_ERRORS_OPT);
+ options.addOption(HELP_OPT);
+ return options;
+ }
+
+ /**
+ * Parses the commandline arguments, throws IllegalStateException if mandatory arguments are
+ * missing.
+ *
+ * @param args supplied command line arguments
+ * @return the parsed command line
+ */
+ protected CommandLine parseOptions(String[] args) {
+
+ Options options = getOptions();
+
+ CommandLineParser parser = new PosixParser();
+ CommandLine cmdLine = null;
+ try {
+ cmdLine = parser.parse(options, args);
+ } catch (ParseException e) {
+ printHelpAndExit("Error parsing command line options: " + e.getMessage(), options);
+ }
+
+ if (cmdLine.hasOption(HELP_OPT.getOpt())) {
+ printHelpAndExit(options, 0);
+ }
+
+ if (!cmdLine.hasOption(TABLE_NAME_OPT.getOpt())) {
+ throw new IllegalStateException(TABLE_NAME_OPT.getLongOpt() + " is a mandatory " +
+ "parameter");
+ }
+
+ if (!cmdLine.getArgList().isEmpty()) {
+ throw new IllegalStateException("Got unexpected extra parameters: "
+ + cmdLine.getArgList());
+ }
+
+ if (!cmdLine.hasOption(INPUT_PATH_OPT.getOpt())) {
+ throw new IllegalStateException(INPUT_PATH_OPT.getLongOpt() + " is a mandatory " +
+ "parameter");
+ }
+
+ return cmdLine;
+ }
+
+ private void printHelpAndExit(String errorMessage, Options options) {
+ System.err.println(errorMessage);
+ printHelpAndExit(options, 1);
+ }
+
+ private void printHelpAndExit(Options options, int exitCode) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("help", options);
+ System.exit(exitCode);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+
+ Configuration conf = HBaseConfiguration.create(getConf());
+
+ CommandLine cmdLine = null;
+ try {
+ cmdLine = parseOptions(args);
+ } catch (IllegalStateException e) {
+ printHelpAndExit(e.getMessage(), getOptions());
+ }
+ return loadData(conf, cmdLine);
+ }
+
+ private int loadData(Configuration conf, CommandLine cmdLine) throws SQLException,
+ InterruptedException, ExecutionException, ClassNotFoundException {
+ String tableName = cmdLine.getOptionValue(TABLE_NAME_OPT.getOpt());
+ String schemaName = cmdLine.getOptionValue(SCHEMA_NAME_OPT.getOpt());
+ String indexTableName = cmdLine.getOptionValue(INDEX_TABLE_NAME_OPT.getOpt());
+ String qualifiedTableName = getQualifiedTableName(schemaName, tableName);
+ String qualifiedIndexTableName = null;
+ if (indexTableName != null){
+ qualifiedIndexTableName = getQualifiedTableName(schemaName, indexTableName);
+ }
+
+ if (cmdLine.hasOption(ZK_QUORUM_OPT.getOpt())) {
+ // ZK_QUORUM_OPT is optional, but if it's there, use it for both the conn and the job.
+ String zkQuorum = cmdLine.getOptionValue(ZK_QUORUM_OPT.getOpt());
+ PhoenixDriver.ConnectionInfo info = PhoenixDriver.ConnectionInfo.create(zkQuorum);
+ LOG.info("Configuring HBase connection to {}", info);
+ for (Map.Entry<String,String> entry : info.asProps()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting {} = {}", entry.getKey(), entry.getValue());
+ }
+ conf.set(entry.getKey(), entry.getValue());
+ }
+ }
+
+ final Connection conn = QueryUtil.getConnection(conf);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Reading columns from {} :: {}", ((PhoenixConnection) conn).getURL(),
+ qualifiedTableName);
+ }
+ List<ColumnInfo> importColumns = buildImportColumns(conn, cmdLine, qualifiedTableName);
+ Preconditions.checkNotNull(importColumns);
+ Preconditions.checkArgument(!importColumns.isEmpty(), "Column info list is empty");
+ FormatToKeyValueMapper.configureColumnInfoList(conf, importColumns);
+ boolean ignoreInvalidRows = cmdLine.hasOption(IGNORE_ERRORS_OPT.getOpt());
+ conf.setBoolean(FormatToKeyValueMapper.IGNORE_INVALID_ROW_CONFKEY, ignoreInvalidRows);
+ conf.set(FormatToKeyValueMapper.TABLE_NAME_CONFKEY, tableName);
+
+ // give subclasses their hook
+ configureOptions(cmdLine, importColumns, conf);
+ try {
+ validateTable(conn, schemaName, tableName);
+ } finally {
+ conn.close();
+ }
+
+ final Path inputPath = new Path(cmdLine.getOptionValue(INPUT_PATH_OPT.getOpt()));
+ final Path outputPath;
+ if (cmdLine.hasOption(OUTPUT_PATH_OPT.getOpt())) {
+ outputPath = new Path(cmdLine.getOptionValue(OUTPUT_PATH_OPT.getOpt()));
+ } else {
+ outputPath = new Path("/tmp/" + UUID.randomUUID());
+ }
+
+ List<TargetTableRef> tablesToBeLoaded = new ArrayList<TargetTableRef>();
+ tablesToBeLoaded.add(new TargetTableRef(qualifiedTableName));
+ // using conn after it's been closed... o.O
+ tablesToBeLoaded.addAll(getIndexTables(conn, schemaName, qualifiedTableName));
+
+ // When loading a single index table, check index table name is correct
+ if (qualifiedIndexTableName != null){
+ TargetTableRef targetIndexRef = null;
+ for (TargetTableRef tmpTable : tablesToBeLoaded){
+ if (tmpTable.getLogicalName().compareToIgnoreCase(qualifiedIndexTableName) == 0) {
+ targetIndexRef = tmpTable;
+ break;
+ }
+ }
+ if (targetIndexRef == null){
+ throw new IllegalStateException("Bulk Loader error: index table " +
+ qualifiedIndexTableName + " doesn't exist");
+ }
+ tablesToBeLoaded.clear();
+ tablesToBeLoaded.add(targetIndexRef);
+ }
+
+ return submitJob(conf, tableName, inputPath, outputPath, tablesToBeLoaded);
+ }
+
+ /**
+ * Submits the jobs to the cluster.
+ * Loads the HFiles onto the respective tables.
+ */
+ public int submitJob(final Configuration conf, final String qualifiedTableName, final Path inputPath,
+ final Path outputPath , List<TargetTableRef> tablesToBeLoaded) {
+ try {
+ Job job = new Job(conf, "Phoenix MapReduce import for " + qualifiedTableName);
+ FileInputFormat.addInputPath(job, inputPath);
+ FileOutputFormat.setOutputPath(job, outputPath);
+
+ job.setInputFormatClass(TextInputFormat.class);
+ job.setMapOutputKeyClass(TableRowkeyPair.class);
+ job.setMapOutputValueClass(KeyValue.class);
+ job.setOutputKeyClass(TableRowkeyPair.class);
+ job.setOutputValueClass(KeyValue.class);
+ job.setReducerClass(FormatToKeyValueReducer.class);
+
+ MultiHfileOutputFormat.configureIncrementalLoad(job, tablesToBeLoaded);
+
+ final String tableNamesAsJson = TargetTableRefFunctions.NAMES_TO_JSON.apply(tablesToBeLoaded);
+ job.getConfiguration().set(FormatToKeyValueMapper.TABLE_NAMES_CONFKEY,tableNamesAsJson);
+
+ // give subclasses their hook
+ setupJob(job);
+
+ LOG.info("Running MapReduce import job from {} to {}", inputPath, outputPath);
+ boolean success = job.waitForCompletion(true);
+
+ if (success) {
+ LOG.info("Loading HFiles from {}", outputPath);
+ completebulkload(conf,outputPath,tablesToBeLoaded);
+ }
+
+ LOG.info("Removing output directory {}", outputPath);
+ if (!FileSystem.get(conf).delete(outputPath, true)) {
+ LOG.error("Removing output directory {} failed", outputPath);
+ }
+ return 0;
+ } catch(Exception e) {
+ LOG.error("Error {} occurred submitting BulkLoad ",e.getMessage());
+ return -1;
+ }
+
+ }
+
+ private void completebulkload(Configuration conf,Path outputPath , List<TargetTableRef> tablesToBeLoaded) throws Exception {
+ for(TargetTableRef table : tablesToBeLoaded) {
+ LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
+ String tableName = table.getPhysicalName();
+ Path tableOutputPath = new Path(outputPath,tableName);
+ HTable htable = new HTable(conf,tableName);
+ LOG.info("Loading HFiles for {} from {}", tableName , tableOutputPath);
+ loader.doBulkLoad(tableOutputPath, htable);
+ LOG.info("Incremental load complete for table=" + tableName);
+ }
+ }
+
+ /**
+ * Build up the list of columns to be imported. The list is taken from the command line if
+ * present, otherwise it is taken from the table description.
+ *
+ * @param conn connection to Phoenix
+ * @param cmdLine supplied command line options
+ * @param qualifiedTableName table name (possibly with schema) of the table to be imported
+ * @return the list of columns to be imported
+ */
+ List<ColumnInfo> buildImportColumns(Connection conn, CommandLine cmdLine,
+ String qualifiedTableName) throws SQLException {
+ List<String> userSuppliedColumnNames = null;
+ if (cmdLine.hasOption(IMPORT_COLUMNS_OPT.getOpt())) {
+ userSuppliedColumnNames = Lists.newArrayList(
+ Splitter.on(",").trimResults().split
+ (cmdLine.getOptionValue(IMPORT_COLUMNS_OPT.getOpt())));
+ }
+ return SchemaUtil.generateColumnInfo(
+ conn, qualifiedTableName, userSuppliedColumnNames, true);
+ }
+
+ /**
+ * Calculate the HBase HTable name for which the import is to be done.
+ *
+ * @param schemaName import schema name, can be null
+ * @param tableName import table name
+ * @return the byte representation of the import HTable
+ */
+ @VisibleForTesting
+ static String getQualifiedTableName(String schemaName, String tableName) {
+ if (schemaName != null) {
+ return String.format("%s.%s", SchemaUtil.normalizeIdentifier(schemaName),
+ SchemaUtil.normalizeIdentifier(tableName));
+ } else {
+ return SchemaUtil.normalizeIdentifier(tableName);
+ }
+ }
+
+ /**
+ * Perform any required validation on the table being bulk loaded into:
+ * - ensure no column family names start with '_', as they'd be ignored leading to problems.
+ * @throws java.sql.SQLException
+ */
+ private void validateTable(Connection conn, String schemaName,
+ String tableName) throws SQLException {
+
+ ResultSet rs = conn.getMetaData().getColumns(
+ null, StringUtil.escapeLike(schemaName),
+ StringUtil.escapeLike(tableName), null);
+ while (rs.next()) {
+ String familyName = rs.getString(PhoenixDatabaseMetaData.COLUMN_FAMILY);
+ if (familyName != null && familyName.startsWith("_")) {
+ if (QueryConstants.DEFAULT_COLUMN_FAMILY.equals(familyName)) {
+ throw new IllegalStateException(
+ "Bulk Loader error: All column names that are not part of the " +
+ "primary key constraint must be prefixed with a column family " +
+ "name (i.e. f.my_column VARCHAR)");
+ } else {
+ throw new IllegalStateException("Bulk Loader error: Column family name " +
+ "must not start with '_': " + familyName);
+ }
+ }
+ }
+ rs.close();
+ }
+
+ /**
+ * Get the index tables of current data table
+ * @throws java.sql.SQLException
+ */
+ private List<TargetTableRef> getIndexTables(Connection conn, String schemaName, String qualifiedTableName)
+ throws SQLException {
+ PTable table = PhoenixRuntime.getTable(conn, qualifiedTableName);
+ List<TargetTableRef> indexTables = new ArrayList<TargetTableRef>();
+ for(PTable indexTable : table.getIndexes()){
+ if (indexTable.getIndexType() == PTable.IndexType.LOCAL) {
+ throw new UnsupportedOperationException("Local indexes not supported by Bulk Loader");
+ /*indexTables.add(
+ new TargetTableRef(getQualifiedTableName(schemaName,
+ indexTable.getTableName().getString()),
+ MetaDataUtil.getLocalIndexTableName(qualifiedTableName))); */
+ } else {
+ indexTables.add(new TargetTableRef(getQualifiedTableName(schemaName,
+ indexTable.getTableName().getString())));
+ }
+ }
+ return indexTables;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkImportUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkImportUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkImportUtil.java
index 6d77cd5..bdc67f9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkImportUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkImportUtil.java
@@ -17,15 +17,11 @@
*/
package org.apache.phoenix.mapreduce;
-import java.util.List;
-
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.util.Base64;
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
-import org.apache.phoenix.util.ColumnInfo;
-import com.google.common.base.Preconditions;
+import com.google.common.annotations.VisibleForTesting;
/**
* Collection of utility methods for setting up bulk import jobs.
@@ -36,29 +32,19 @@ public class CsvBulkImportUtil {
* Configure a job configuration for a bulk CSV import.
*
* @param conf job configuration to be set up
- * @param tableName name of the table to be imported to, can include a schema name
* @param fieldDelimiter field delimiter character for the CSV input
* @param quoteChar quote character for the CSV input
* @param escapeChar escape character for the CSV input
* @param arrayDelimiter array delimiter character, can be null
- * @param columnInfoList list of columns to be imported
- * @param ignoreInvalidRows flag to ignore invalid input rows
*/
- public static void initCsvImportJob(Configuration conf, String tableName, char fieldDelimiter, char quoteChar, char escapeChar,
- String arrayDelimiter, List<ColumnInfo> columnInfoList, boolean ignoreInvalidRows) {
-
- Preconditions.checkNotNull(tableName);
- Preconditions.checkNotNull(columnInfoList);
- Preconditions.checkArgument(!columnInfoList.isEmpty(), "Column info list is empty");
- conf.set(CsvToKeyValueMapper.TABLE_NAME_CONFKEY, tableName);
+ public static void initCsvImportJob(Configuration conf, char fieldDelimiter, char quoteChar,
+ char escapeChar, String arrayDelimiter) {
setChar(conf, CsvToKeyValueMapper.FIELD_DELIMITER_CONFKEY, fieldDelimiter);
setChar(conf, CsvToKeyValueMapper.QUOTE_CHAR_CONFKEY, quoteChar);
setChar(conf, CsvToKeyValueMapper.ESCAPE_CHAR_CONFKEY, escapeChar);
if (arrayDelimiter != null) {
conf.set(CsvToKeyValueMapper.ARRAY_DELIMITER_CONFKEY, arrayDelimiter);
}
- CsvToKeyValueMapper.configureColumnInfoList(conf, columnInfoList);
- conf.setBoolean(CsvToKeyValueMapper.IGNORE_INVALID_ROW_CONFKEY, ignoreInvalidRows);
}
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
index 20f05ff..e0b083e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
@@ -17,364 +17,37 @@
*/
package org.apache.phoenix.mapreduce;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ExecutionException;
import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.phoenix.jdbc.PhoenixConnection;
-import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
-import org.apache.phoenix.jdbc.PhoenixDriver;
-import org.apache.phoenix.mapreduce.bulkload.CsvTableRowkeyPair;
-import org.apache.phoenix.query.QueryConstants;
-import org.apache.phoenix.schema.PTable;
-import org.apache.phoenix.schema.PTable.IndexType;
-import org.apache.phoenix.util.CSVCommonsLoader;
import org.apache.phoenix.util.ColumnInfo;
-import org.apache.phoenix.util.PhoenixRuntime;
-import org.apache.phoenix.util.QueryUtil;
-import org.apache.phoenix.util.SchemaUtil;
-import org.apache.phoenix.util.StringUtil;
-import org.codehaus.jackson.annotate.JsonCreator;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Splitter;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+public class CsvBulkLoadTool extends AbstractBulkLoadTool {
-/**
- * Base tool for running MapReduce-based ingests of data.
- */
-@SuppressWarnings("deprecation")
-public class CsvBulkLoadTool extends Configured implements Tool {
-
- private static final Logger LOG = LoggerFactory.getLogger(CsvBulkLoadTool.class);
-
- static final Option ZK_QUORUM_OPT = new Option("z", "zookeeper", true, "Supply zookeeper connection details (optional)");
- static final Option INPUT_PATH_OPT = new Option("i", "input", true, "Input CSV path (mandatory)");
- static final Option OUTPUT_PATH_OPT = new Option("o", "output", true, "Output path for temporary HFiles (optional)");
- static final Option SCHEMA_NAME_OPT = new Option("s", "schema", true, "Phoenix schema name (optional)");
- static final Option TABLE_NAME_OPT = new Option("t", "table", true, "Phoenix table name (mandatory)");
- static final Option INDEX_TABLE_NAME_OPT = new Option("it", "index-table", true, "Phoenix index table name when just loading this particualar index table");
static final Option DELIMITER_OPT = new Option("d", "delimiter", true, "Input delimiter, defaults to comma");
static final Option QUOTE_OPT = new Option("q", "quote", true, "Supply a custom phrase delimiter, defaults to double quote character");
static final Option ESCAPE_OPT = new Option("e", "escape", true, "Supply a custom escape character, default is a backslash");
static final Option ARRAY_DELIMITER_OPT = new Option("a", "array-delimiter", true, "Array element delimiter (optional)");
- static final Option IMPORT_COLUMNS_OPT = new Option("c", "import-columns", true, "Comma-separated list of columns to be imported");
- static final Option IGNORE_ERRORS_OPT = new Option("g", "ignore-errors", false, "Ignore input errors");
- static final Option HELP_OPT = new Option("h", "help", false, "Show this help and quit");
-
- public static void main(String[] args) throws Exception {
- int exitStatus = ToolRunner.run(new CsvBulkLoadTool(), args);
- System.exit(exitStatus);
- }
-
- /**
- * Parses the commandline arguments, throws IllegalStateException if mandatory arguments are
- * missing.
- *
- * @param args supplied command line arguments
- * @return the parsed command line
- */
- CommandLine parseOptions(String[] args) {
-
- Options options = getOptions();
-
- CommandLineParser parser = new PosixParser();
- CommandLine cmdLine = null;
- try {
- cmdLine = parser.parse(options, args);
- } catch (ParseException e) {
- printHelpAndExit("Error parsing command line options: " + e.getMessage(), options);
- }
-
- if (cmdLine.hasOption(HELP_OPT.getOpt())) {
- printHelpAndExit(options, 0);
- }
-
- if (!cmdLine.hasOption(TABLE_NAME_OPT.getOpt())) {
- throw new IllegalStateException(TABLE_NAME_OPT.getLongOpt() + " is a mandatory " +
- "parameter");
- }
-
- if (!cmdLine.getArgList().isEmpty()) {
- throw new IllegalStateException("Got unexpected extra parameters: "
- + cmdLine.getArgList());
- }
-
- if (!cmdLine.hasOption(INPUT_PATH_OPT.getOpt())) {
- throw new IllegalStateException(INPUT_PATH_OPT.getLongOpt() + " is a mandatory " +
- "parameter");
- }
- return cmdLine;
- }
-
- private Options getOptions() {
- Options options = new Options();
- options.addOption(INPUT_PATH_OPT);
- options.addOption(TABLE_NAME_OPT);
- options.addOption(INDEX_TABLE_NAME_OPT);
- options.addOption(ZK_QUORUM_OPT);
- options.addOption(OUTPUT_PATH_OPT);
- options.addOption(SCHEMA_NAME_OPT);
+ @Override
+ protected Options getOptions() {
+ Options options = super.getOptions();
options.addOption(DELIMITER_OPT);
options.addOption(QUOTE_OPT);
options.addOption(ESCAPE_OPT);
options.addOption(ARRAY_DELIMITER_OPT);
- options.addOption(IMPORT_COLUMNS_OPT);
- options.addOption(IGNORE_ERRORS_OPT);
- options.addOption(HELP_OPT);
return options;
}
-
- private void printHelpAndExit(String errorMessage, Options options) {
- System.err.println(errorMessage);
- printHelpAndExit(options, 1);
- }
-
- private void printHelpAndExit(Options options, int exitCode) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("help", options);
- System.exit(exitCode);
- }
-
@Override
- public int run(String[] args) throws Exception {
-
- Configuration conf = HBaseConfiguration.create(getConf());
-
- CommandLine cmdLine = null;
- try {
- cmdLine = parseOptions(args);
- } catch (IllegalStateException e) {
- printHelpAndExit(e.getMessage(), getOptions());
- }
- return loadData(conf, cmdLine);
- }
-
- private int loadData(Configuration conf, CommandLine cmdLine) throws SQLException,
- InterruptedException, ExecutionException, ClassNotFoundException {
- String tableName = cmdLine.getOptionValue(TABLE_NAME_OPT.getOpt());
- String schemaName = cmdLine.getOptionValue(SCHEMA_NAME_OPT.getOpt());
- String indexTableName = cmdLine.getOptionValue(INDEX_TABLE_NAME_OPT.getOpt());
- String qualifiedTableName = getQualifiedTableName(schemaName, tableName);
- String qualifiedIndexTableName = null;
- if (indexTableName != null){
- qualifiedIndexTableName = getQualifiedTableName(schemaName, indexTableName);
- }
-
- if (cmdLine.hasOption(ZK_QUORUM_OPT.getOpt())) {
- // ZK_QUORUM_OPT is optional, but if it's there, use it for both the conn and the job.
- String zkQuorum = cmdLine.getOptionValue(ZK_QUORUM_OPT.getOpt());
- PhoenixDriver.ConnectionInfo info = PhoenixDriver.ConnectionInfo.create(zkQuorum);
- LOG.info("Configuring HBase connection to {}", info);
- for (Map.Entry<String,String> entry : info.asProps()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Setting {} = {}", entry.getKey(), entry.getValue());
- }
- conf.set(entry.getKey(), entry.getValue());
- }
- }
-
- final Connection conn = QueryUtil.getConnection(conf);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Reading columns from {} :: {}", ((PhoenixConnection) conn).getURL(),
- qualifiedTableName);
- }
- List<ColumnInfo> importColumns = buildImportColumns(conn, cmdLine, qualifiedTableName);
- configureOptions(cmdLine, importColumns, conf);
- try {
- validateTable(conn, schemaName, tableName);
- } finally {
- conn.close();
- }
-
- final Path inputPath = new Path(cmdLine.getOptionValue(INPUT_PATH_OPT.getOpt()));
- final Path outputPath;
- if (cmdLine.hasOption(OUTPUT_PATH_OPT.getOpt())) {
- outputPath = new Path(cmdLine.getOptionValue(OUTPUT_PATH_OPT.getOpt()));
- } else {
- outputPath = new Path("/tmp/" + UUID.randomUUID());
- }
-
- List<TargetTableRef> tablesToBeLoaded = new ArrayList<TargetTableRef>();
- tablesToBeLoaded.add(new TargetTableRef(qualifiedTableName));
- // using conn after it's been closed... o.O
- tablesToBeLoaded.addAll(getIndexTables(conn, schemaName, qualifiedTableName));
-
- // When loading a single index table, check index table name is correct
- if (qualifiedIndexTableName != null){
- TargetTableRef targetIndexRef = null;
- for (TargetTableRef tmpTable : tablesToBeLoaded){
- if (tmpTable.getLogicalName().compareToIgnoreCase(qualifiedIndexTableName) == 0) {
- targetIndexRef = tmpTable;
- break;
- }
- }
- if (targetIndexRef == null){
- throw new IllegalStateException("CSV Bulk Loader error: index table " +
- qualifiedIndexTableName + " doesn't exist");
- }
- tablesToBeLoaded.clear();
- tablesToBeLoaded.add(targetIndexRef);
- }
-
- return submitJob(conf, tableName, inputPath, outputPath, tablesToBeLoaded);
- }
-
- /**
- * Submits the jobs to the cluster.
- * Loads the HFiles onto the respective tables.
- * @param configuration
- * @param qualifiedTableName
- * @param inputPath
- * @param outputPath
- * @param tablesToBeoaded
- * @return status
- */
- public int submitJob(final Configuration conf, final String qualifiedTableName, final Path inputPath,
- final Path outputPath , List<TargetTableRef> tablesToBeLoaded) {
- try {
- Job job = new Job(conf, "Phoenix MapReduce import for " + qualifiedTableName);
-
- // Allow overriding the job jar setting by using a -D system property at startup
- if (job.getJar() == null) {
- job.setJarByClass(CsvToKeyValueMapper.class);
- }
- job.setInputFormatClass(TextInputFormat.class);
- FileInputFormat.addInputPath(job, inputPath);
- FileOutputFormat.setOutputPath(job, outputPath);
- job.setMapperClass(CsvToKeyValueMapper.class);
- job.setMapOutputKeyClass(CsvTableRowkeyPair.class);
- job.setMapOutputValueClass(KeyValue.class);
- job.setOutputKeyClass(CsvTableRowkeyPair.class);
- job.setOutputValueClass(KeyValue.class);
- job.setReducerClass(CsvToKeyValueReducer.class);
-
- MultiHfileOutputFormat.configureIncrementalLoad(job, tablesToBeLoaded);
-
- final String tableNamesAsJson = TargetTableRefFunctions.NAMES_TO_JSON.apply(tablesToBeLoaded);
- job.getConfiguration().set(CsvToKeyValueMapper.TABLE_NAMES_CONFKEY,tableNamesAsJson);
-
- LOG.info("Running MapReduce import job from {} to {}", inputPath, outputPath);
- boolean success = job.waitForCompletion(true);
-
- if (success) {
- LOG.info("Loading HFiles from {}", outputPath);
- completebulkload(conf,outputPath,tablesToBeLoaded);
- }
-
- LOG.info("Removing output directory {}", outputPath);
- if (!FileSystem.get(conf).delete(outputPath, true)) {
- LOG.error("Removing output directory {} failed", outputPath);
- }
- return 0;
- } catch(Exception e) {
- LOG.error("Error {} occurred submitting CSVBulkLoad ",e.getMessage());
- return -1;
- }
-
- }
-
- /**
- * bulkload HFiles .
- * @param conf
- * @param outputPath
- * @param tablesToBeLoaded
- * @throws Exception
- */
- private void completebulkload(Configuration conf,Path outputPath , List<TargetTableRef> tablesToBeLoaded) throws Exception {
- for(TargetTableRef table : tablesToBeLoaded) {
- LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
- String tableName = table.getPhysicalName();
- Path tableOutputPath = new Path(outputPath,tableName);
- HTable htable = new HTable(conf,tableName);
- LOG.info("Loading HFiles for {} from {}", tableName , tableOutputPath);
- loader.doBulkLoad(tableOutputPath, htable);
- LOG.info("Incremental load complete for table=" + tableName);
- }
- }
-
- /**
- * Build up the list of columns to be imported. The list is taken from the command line if
- * present, otherwise it is taken from the table description.
- *
- * @param conn connection to Phoenix
- * @param cmdLine supplied command line options
- * @param qualifiedTableName table name (possibly with schema) of the table to be imported
- * @return the list of columns to be imported
- */
- List<ColumnInfo> buildImportColumns(Connection conn, CommandLine cmdLine,
- String qualifiedTableName) throws SQLException {
- List<String> userSuppliedColumnNames = null;
- if (cmdLine.hasOption(IMPORT_COLUMNS_OPT.getOpt())) {
- userSuppliedColumnNames = Lists.newArrayList(
- Splitter.on(",").trimResults().split
- (cmdLine.getOptionValue(IMPORT_COLUMNS_OPT.getOpt())));
- }
- return CSVCommonsLoader.generateColumnInfo(
- conn, qualifiedTableName, userSuppliedColumnNames, true);
- }
-
- /**
- * Calculate the HBase HTable name for which the import is to be done.
- *
- * @param schemaName import schema name, can be null
- * @param tableName import table name
- * @return the byte representation of the import HTable
- */
- @VisibleForTesting
- static String getQualifiedTableName(String schemaName, String tableName) {
- if (schemaName != null) {
- return String.format("%s.%s", SchemaUtil.normalizeIdentifier(schemaName),
- SchemaUtil.normalizeIdentifier(tableName));
- } else {
- return SchemaUtil.normalizeIdentifier(tableName);
- }
- }
-
- /**
- * Set configuration values based on parsed command line options.
- *
- * @param cmdLine supplied command line options
- * @param importColumns descriptors of columns to be imported
- * @param conf job configuration
- */
- private static void configureOptions(CommandLine cmdLine, List<ColumnInfo> importColumns,
- Configuration conf) throws SQLException {
+ protected void configureOptions(CommandLine cmdLine, List<ColumnInfo> importColumns,
+ Configuration conf) throws SQLException {
// we don't parse ZK_QUORUM_OPT here because we need it in order to
// create the connection we need to build importColumns.
@@ -408,178 +81,23 @@ public class CsvBulkLoadTool extends Configured implements Tool {
CsvBulkImportUtil.initCsvImportJob(
conf,
- getQualifiedTableName(
- cmdLine.getOptionValue(SCHEMA_NAME_OPT.getOpt()),
- cmdLine.getOptionValue(TABLE_NAME_OPT.getOpt())),
delimiterChar,
quoteChar,
escapeChar,
- cmdLine.getOptionValue(ARRAY_DELIMITER_OPT.getOpt()),
- importColumns,
- cmdLine.hasOption(IGNORE_ERRORS_OPT.getOpt()));
+ cmdLine.getOptionValue(ARRAY_DELIMITER_OPT.getOpt()));
}
- /**
- * Perform any required validation on the table being bulk loaded into:
- * - ensure no column family names start with '_', as they'd be ignored leading to problems.
- * @throws java.sql.SQLException
- */
- private void validateTable(Connection conn, String schemaName,
- String tableName) throws SQLException {
-
- ResultSet rs = conn.getMetaData().getColumns(
- null, StringUtil.escapeLike(schemaName),
- StringUtil.escapeLike(tableName), null);
- while (rs.next()) {
- String familyName = rs.getString(PhoenixDatabaseMetaData.COLUMN_FAMILY);
- if (familyName != null && familyName.startsWith("_")) {
- if (QueryConstants.DEFAULT_COLUMN_FAMILY.equals(familyName)) {
- throw new IllegalStateException(
- "CSV Bulk Loader error: All column names that are not part of the " +
- "primary key constraint must be prefixed with a column family " +
- "name (i.e. f.my_column VARCHAR)");
- } else {
- throw new IllegalStateException("CSV Bulk Loader error: Column family name " +
- "must not start with '_': " + familyName);
- }
- }
- }
- rs.close();
- }
-
- /**
- * Get the index tables of current data table
- * @throws java.sql.SQLException
- */
- private List<TargetTableRef> getIndexTables(Connection conn, String schemaName, String qualifiedTableName)
- throws SQLException {
- PTable table = PhoenixRuntime.getTable(conn, qualifiedTableName);
- List<TargetTableRef> indexTables = new ArrayList<TargetTableRef>();
- for(PTable indexTable : table.getIndexes()){
- if (indexTable.getIndexType() == IndexType.LOCAL) {
- throw new UnsupportedOperationException("Local indexes not supported by CSV Bulk Loader");
- /*indexTables.add(
- new TargetTableRef(getQualifiedTableName(schemaName,
- indexTable.getTableName().getString()),
- MetaDataUtil.getLocalIndexTableName(qualifiedTableName))); */
- } else {
- indexTables.add(new TargetTableRef(getQualifiedTableName(schemaName,
- indexTable.getTableName().getString())));
- }
+ @Override
+ protected void setupJob(Job job) {
+ // Allow overriding the job jar setting by using a -D system property at startup
+ if (job.getJar() == null) {
+ job.setJarByClass(CsvToKeyValueMapper.class);
}
- return indexTables;
+ job.setMapperClass(CsvToKeyValueMapper.class);
}
- /**
- * Represents the logical and physical name of a single table to which data is to be loaded.
- *
- * This class exists to allow for the difference between HBase physical table names and
- * Phoenix logical table names.
- */
- static class TargetTableRef {
-
- @JsonProperty
- private final String logicalName;
-
- @JsonProperty
- private final String physicalName;
-
- @JsonProperty
- private Map<String,String> configuration = Maps.newHashMap();
-
- private TargetTableRef(String name) {
- this(name, name);
- }
-
- @JsonCreator
- private TargetTableRef(@JsonProperty("logicalName") String logicalName, @JsonProperty("physicalName") String physicalName) {
- this.logicalName = logicalName;
- this.physicalName = physicalName;
- }
-
- public String getLogicalName() {
- return logicalName;
- }
-
- public String getPhysicalName() {
- return physicalName;
- }
-
- public Map<String, String> getConfiguration() {
- return configuration;
- }
-
- public void setConfiguration(Map<String, String> configuration) {
- this.configuration = configuration;
- }
+ public static void main(String[] args) throws Exception {
+ int exitStatus = ToolRunner.run(new CsvBulkLoadTool(), args);
+ System.exit(exitStatus);
}
-
- /**
- * Utility functions to get/put json.
- *
- */
- static class TargetTableRefFunctions {
-
- public static Function<TargetTableRef,String> TO_JSON = new Function<TargetTableRef,String>() {
-
- @Override
- public String apply(TargetTableRef input) {
- try {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.writeValueAsString(input);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- }
- };
-
- public static Function<String,TargetTableRef> FROM_JSON = new Function<String,TargetTableRef>() {
-
- @Override
- public TargetTableRef apply(String json) {
- try {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.readValue(json, TargetTableRef.class);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- }
- };
-
- public static Function<List<TargetTableRef>,String> NAMES_TO_JSON = new Function<List<TargetTableRef>,String>() {
-
- @Override
- public String apply(List<TargetTableRef> input) {
- try {
- List<String> tableNames = Lists.newArrayListWithCapacity(input.size());
- for(TargetTableRef table : input) {
- tableNames.add(table.getPhysicalName());
- }
- ObjectMapper mapper = new ObjectMapper();
- return mapper.writeValueAsString(tableNames);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- }
- };
-
- public static Function<String,List<String>> NAMES_FROM_JSON = new Function<String,List<String>>() {
-
- @SuppressWarnings("unchecked")
- @Override
- public List<String> apply(String json) {
- try {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.readValue(json, ArrayList.class);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- }
- };
- }
-
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
index c3b5a7d..5a5d378 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
@@ -19,46 +19,20 @@ package org.apache.phoenix.mapreduce;
import java.io.IOException;
import java.io.StringReader;
-import java.sql.SQLException;
-import java.util.Iterator;
import java.util.List;
-import java.util.Map.Entry;
-import java.util.Properties;
-
-import javax.annotation.Nullable;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.phoenix.jdbc.PhoenixConnection;
-import org.apache.phoenix.mapreduce.CsvBulkLoadTool.TargetTableRefFunctions;
-import org.apache.phoenix.mapreduce.bulkload.CsvTableRowkeyPair;
-import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
import org.apache.phoenix.util.CSVCommonsLoader;
import org.apache.phoenix.util.ColumnInfo;
-import org.apache.phoenix.util.PhoenixRuntime;
-import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.UpsertExecutor;
import org.apache.phoenix.util.csv.CsvUpsertExecutor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
-import com.google.common.base.Splitter;
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
/**
* MapReduce mapper that converts CSV input lines into KeyValues that can be written to HFiles.
@@ -67,12 +41,7 @@ import com.google.common.collect.Lists;
* extracting the created KeyValues and rolling back the statement execution before it is
* committed to HBase.
*/
-public class CsvToKeyValueMapper extends Mapper<LongWritable,Text,CsvTableRowkeyPair,
- KeyValue> {
-
- private static final Logger LOG = LoggerFactory.getLogger(CsvToKeyValueMapper.class);
-
- private static final String COUNTER_GROUP_NAME = "Phoenix MapReduce Import";
+public class CsvToKeyValueMapper extends FormatToKeyValueMapper<CSVRecord> {
/** Configuration key for the field delimiter for input csv records */
public static final String FIELD_DELIMITER_CONFKEY = "phoenix.mapreduce.import.fielddelimiter";
@@ -86,117 +55,25 @@ public class CsvToKeyValueMapper extends Mapper<LongWritable,Text,CsvTableRowkey
/** Configuration key for the array element delimiter for input arrays */
public static final String ARRAY_DELIMITER_CONFKEY = "phoenix.mapreduce.import.arraydelimiter";
- /** Configuration key for the name of the output table */
- public static final String TABLE_NAME_CONFKEY = "phoenix.mapreduce.import.tablename";
-
- /** Configuration key for the name of the output index table */
- public static final String INDEX_TABLE_NAME_CONFKEY = "phoenix.mapreduce.import.indextablename";
-
- /** Configuration key for the columns to be imported */
- public static final String COLUMN_INFO_CONFKEY = "phoenix.mapreduce.import.columninfos";
+ private CsvLineParser lineParser;
- /** Configuration key for the flag to ignore invalid rows */
- public static final String IGNORE_INVALID_ROW_CONFKEY = "phoenix.mapreduce.import.ignoreinvalidrow";
-
- /** Configuration key for the table names */
- public static final String TABLE_NAMES_CONFKEY = "phoenix.mapreduce.import.tablenames";
-
- /** Configuration key for the table configurations */
- public static final String TABLE_CONFIG_CONFKEY = "phoenix.mapreduce.import.table.config";
-
- private PhoenixConnection conn;
- private CsvUpsertExecutor csvUpsertExecutor;
- private MapperUpsertListener upsertListener;
- private CsvLineParser csvLineParser;
- private ImportPreUpsertKeyValueProcessor preUpdateProcessor;
- private List<String> tableNames;
+ @Override
+ protected LineParser<CSVRecord> getLineParser() {
+ return lineParser;
+ }
@Override
protected void setup(Context context) throws IOException, InterruptedException {
-
Configuration conf = context.getConfiguration();
-
- // pass client configuration into driver
- Properties clientInfos = new Properties();
- Iterator<Entry<String, String>> iterator = conf.iterator();
- while(iterator.hasNext()) {
- Entry<String,String> entry = iterator.next();
- clientInfos.setProperty(entry.getKey(), entry.getValue());
- }
-
- try {
- conn = (PhoenixConnection) QueryUtil.getConnection(clientInfos, conf);
- } catch (SQLException | ClassNotFoundException e) {
- throw new RuntimeException(e);
- }
-
- final String tableNamesConf = conf.get(TABLE_NAMES_CONFKEY);
- tableNames = TargetTableRefFunctions.NAMES_FROM_JSON.apply(tableNamesConf);
-
- upsertListener = new MapperUpsertListener(
- context, conf.getBoolean(IGNORE_INVALID_ROW_CONFKEY, true));
- csvUpsertExecutor = buildUpsertExecutor(conf);
- csvLineParser = new CsvLineParser(
+ lineParser = new CsvLineParser(
CsvBulkImportUtil.getCharacter(conf, FIELD_DELIMITER_CONFKEY),
CsvBulkImportUtil.getCharacter(conf, QUOTE_CHAR_CONFKEY),
CsvBulkImportUtil.getCharacter(conf, ESCAPE_CHAR_CONFKEY));
-
- preUpdateProcessor = PhoenixConfigurationUtil.loadPreUpsertProcessor(conf);
- }
-
- @SuppressWarnings("deprecation")
- @Override
- protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
- try {
- CSVRecord csvRecord = null;
- try {
- csvRecord = csvLineParser.parse(value.toString());
- } catch (IOException e) {
- context.getCounter(COUNTER_GROUP_NAME, "CSV Parser errors").increment(1L);
- }
-
- if (csvRecord == null) {
- context.getCounter(COUNTER_GROUP_NAME, "Empty records").increment(1L);
- return;
- }
- csvUpsertExecutor.execute(ImmutableList.of(csvRecord));
-
- Iterator<Pair<byte[], List<KeyValue>>> uncommittedDataIterator
- = PhoenixRuntime.getUncommittedDataIterator(conn, true);
- while (uncommittedDataIterator.hasNext()) {
- Pair<byte[], List<KeyValue>> kvPair = uncommittedDataIterator.next();
- List<KeyValue> keyValueList = kvPair.getSecond();
- keyValueList = preUpdateProcessor.preUpsert(kvPair.getFirst(), keyValueList);
- byte[] first = kvPair.getFirst();
- for(String tableName : tableNames) {
- if (Bytes.compareTo(Bytes.toBytes(tableName), first) != 0) {
- // skip edits for other tables
- continue;
- }
- for (KeyValue kv : keyValueList) {
- ImmutableBytesWritable outputKey = new ImmutableBytesWritable();
- outputKey.set(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength());
- context.write(new CsvTableRowkeyPair(tableName, outputKey), kv);
- }
- }
- }
- conn.rollback();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- protected void cleanup(Context context) throws IOException, InterruptedException {
- try {
- conn.close();
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
}
@VisibleForTesting
- CsvUpsertExecutor buildUpsertExecutor(Configuration conf) {
+ @Override
+ protected UpsertExecutor<CSVRecord, ?> buildUpsertExecutor(Configuration conf) {
String tableName = conf.get(TABLE_NAME_CONFKEY);
String arraySeparator = conf.get(ARRAY_DELIMITER_CONFKEY,
CSVCommonsLoader.DEFAULT_ARRAY_ELEMENT_SEPARATOR);
@@ -204,78 +81,14 @@ public class CsvToKeyValueMapper extends Mapper<LongWritable,Text,CsvTableRowkey
List<ColumnInfo> columnInfoList = buildColumnInfoList(conf);
- return CsvUpsertExecutor.create(conn, tableName, columnInfoList, upsertListener, arraySeparator);
- }
-
- /**
- * Write the list of to-import columns to a job configuration.
- *
- * @param conf configuration to be written to
- * @param columnInfoList list of ColumnInfo objects to be configured for import
- */
- @VisibleForTesting
- static void configureColumnInfoList(Configuration conf, List<ColumnInfo> columnInfoList) {
- conf.set(COLUMN_INFO_CONFKEY, Joiner.on("|").useForNull("").join(columnInfoList));
- }
-
- /**
- * Build the list of ColumnInfos for the import based on information in the configuration.
- */
- @VisibleForTesting
- static List<ColumnInfo> buildColumnInfoList(Configuration conf) {
-
- return Lists.newArrayList(
- Iterables.transform(
- Splitter.on("|").split(conf.get(COLUMN_INFO_CONFKEY)),
- new Function<String, ColumnInfo>() {
- @Nullable
- @Override
- public ColumnInfo apply(@Nullable String input) {
- if (input.isEmpty()) {
- // An empty string represents a null that was passed in to
- // the configuration, which corresponds to an input column
- // which is to be skipped
- return null;
- }
- return ColumnInfo.fromString(input);
- }
- }));
- }
-
- /**
- * Listener that logs successful upserts and errors to job counters.
- */
- @VisibleForTesting
- static class MapperUpsertListener implements CsvUpsertExecutor.UpsertListener {
-
- private final Context context;
- private final boolean ignoreRecordErrors;
-
- private MapperUpsertListener(Context context, boolean ignoreRecordErrors) {
- this.context = context;
- this.ignoreRecordErrors = ignoreRecordErrors;
- }
-
- @Override
- public void upsertDone(long upsertCount) {
- context.getCounter(COUNTER_GROUP_NAME, "Upserts Done").increment(1L);
- }
-
- @Override
- public void errorOnRecord(CSVRecord csvRecord, Throwable throwable) {
- LOG.error("Error on record " + csvRecord, throwable);
- context.getCounter(COUNTER_GROUP_NAME, "Errors on records").increment(1L);
- if (!ignoreRecordErrors) {
- throw Throwables.propagate(throwable);
- }
- }
+ return new CsvUpsertExecutor(conn, tableName, columnInfoList, upsertListener, arraySeparator);
}
/**
* Parses a single CSV input line, returning a {@code CSVRecord}.
*/
@VisibleForTesting
- static class CsvLineParser {
+ static class CsvLineParser implements LineParser<CSVRecord> {
private final CSVFormat csvFormat;
CsvLineParser(char fieldDelimiter, char quote, char escape) {
@@ -286,6 +99,7 @@ public class CsvToKeyValueMapper extends Mapper<LongWritable,Text,CsvTableRowkey
.withQuote(quote);
}
+ @Override
public CSVRecord parse(String input) throws IOException {
// TODO Creating a new parser for each line seems terribly inefficient but
// there's no public way to parse single lines via commons-csv. We should update
@@ -294,18 +108,4 @@ public class CsvToKeyValueMapper extends Mapper<LongWritable,Text,CsvTableRowkey
return Iterables.getFirst(csvParser, null);
}
}
-
- /**
- * A default implementation of {@code ImportPreUpsertKeyValueProcessor} that is used if no
- * specific class is configured. This implementation simply passes through the KeyValue
- * list that is passed in.
- */
- public static class DefaultImportPreUpsertKeyValueProcessor implements
- ImportPreUpsertKeyValueProcessor {
-
- @Override
- public List<KeyValue> preUpsert(byte[] rowKey, List<KeyValue> keyValues) {
- return keyValues;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueReducer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueReducer.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueReducer.java
deleted file mode 100644
index 7e9c4fd..0000000
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueReducer.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.mapreduce;
-
-import java.io.IOException;
-import java.util.TreeSet;
-
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.mapreduce.KeyValueSortReducer;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.phoenix.mapreduce.bulkload.CsvTableRowkeyPair;
-
-/**
- * Reducer class for the CSVBulkLoad job.
- * Performs similar functionality to {@link KeyValueSortReducer}
- *
- */
-public class CsvToKeyValueReducer extends Reducer<CsvTableRowkeyPair,KeyValue,CsvTableRowkeyPair,KeyValue> {
-
- @Override
- protected void reduce(CsvTableRowkeyPair key, Iterable<KeyValue> values,
- Reducer<CsvTableRowkeyPair, KeyValue, CsvTableRowkeyPair, KeyValue>.Context context)
- throws IOException, InterruptedException {
- TreeSet<KeyValue> map = new TreeSet<KeyValue>(KeyValue.COMPARATOR);
- for (KeyValue kv: values) {
- try {
- map.add(kv.clone());
- } catch (CloneNotSupportedException e) {
- throw new java.io.IOException(e);
- }
- }
- context.setStatus("Read " + map.getClass());
- int index = 0;
- for (KeyValue kv: map) {
- context.write(key, kv);
- if (++index % 100 == 0) context.setStatus("Wrote " + index);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueMapper.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueMapper.java
new file mode 100644
index 0000000..b2e99e5
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueMapper.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.mapreduce.bulkload.TableRowkeyPair;
+import org.apache.phoenix.mapreduce.bulkload.TargetTableRefFunctions;
+import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
+import org.apache.phoenix.util.ColumnInfo;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.UpsertExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+
+/**
+ * Base class for converting some input source format into {@link KeyValue}s of a target
+ * schema. Assumes input format is text-based, with one row per line. Depends on an online cluster
+ * to retrieve {@link ColumnInfo} from the target table.
+ */
+public abstract class FormatToKeyValueMapper<RECORD> extends Mapper<LongWritable, Text, TableRowkeyPair,
+ KeyValue> {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(FormatToKeyValueMapper.class);
+
+ protected static final String COUNTER_GROUP_NAME = "Phoenix MapReduce Import";
+
+ /** Configuration key for the name of the output table */
+ public static final String TABLE_NAME_CONFKEY = "phoenix.mapreduce.import.tablename";
+
+ /** Configuration key for the name of the output index table */
+ public static final String INDEX_TABLE_NAME_CONFKEY = "phoenix.mapreduce.import.indextablename";
+
+ /** Configuration key for the columns to be imported */
+ public static final String COLUMN_INFO_CONFKEY = "phoenix.mapreduce.import.columninfos";
+
+ /** Configuration key for the flag to ignore invalid rows */
+ public static final String IGNORE_INVALID_ROW_CONFKEY = "phoenix.mapreduce.import.ignoreinvalidrow";
+
+ /** Configuration key for the table names */
+ public static final String TABLE_NAMES_CONFKEY = "phoenix.mapreduce.import.tablenames";
+
+ /** Configuration key for the table configurations */
+ public static final String TABLE_CONFIG_CONFKEY = "phoenix.mapreduce.import.table.config";
+
+ /**
+ * Parses a single input line, returning a {@code T}.
+ */
+ public interface LineParser<T> {
+ T parse(String input) throws IOException;
+ }
+
+ protected PhoenixConnection conn;
+ protected UpsertExecutor<RECORD, ?> upsertExecutor;
+ protected ImportPreUpsertKeyValueProcessor preUpdateProcessor;
+ protected List<String> tableNames;
+ protected MapperUpsertListener<RECORD> upsertListener;
+
+ protected abstract UpsertExecutor<RECORD,?> buildUpsertExecutor(Configuration conf);
+ protected abstract LineParser<RECORD> getLineParser();
+
+ @Override
+ protected void setup(Context context) throws IOException, InterruptedException {
+
+ Configuration conf = context.getConfiguration();
+
+ // pass client configuration into driver
+ Properties clientInfos = new Properties();
+ for (Map.Entry<String, String> entry : conf) {
+ clientInfos.setProperty(entry.getKey(), entry.getValue());
+ }
+
+ try {
+ conn = (PhoenixConnection) QueryUtil.getConnection(clientInfos, conf);
+ } catch (SQLException | ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+
+ final String tableNamesConf = conf.get(TABLE_NAMES_CONFKEY);
+ tableNames = TargetTableRefFunctions.NAMES_FROM_JSON.apply(tableNamesConf);
+
+ upsertListener = new MapperUpsertListener<RECORD>(
+ context, conf.getBoolean(IGNORE_INVALID_ROW_CONFKEY, true));
+ upsertExecutor = buildUpsertExecutor(conf);
+ preUpdateProcessor = PhoenixConfigurationUtil.loadPreUpsertProcessor(conf);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
+ try {
+ RECORD record = null;
+ try {
+ record = getLineParser().parse(value.toString());
+ } catch (IOException e) {
+ context.getCounter(COUNTER_GROUP_NAME, "Parser errors").increment(1L);
+ return;
+ }
+
+ if (record == null) {
+ context.getCounter(COUNTER_GROUP_NAME, "Empty records").increment(1L);
+ return;
+ }
+ upsertExecutor.execute(ImmutableList.<RECORD>of(record));
+
+ Iterator<Pair<byte[], List<KeyValue>>> uncommittedDataIterator
+ = PhoenixRuntime.getUncommittedDataIterator(conn, true);
+ while (uncommittedDataIterator.hasNext()) {
+ Pair<byte[], List<KeyValue>> kvPair = uncommittedDataIterator.next();
+ List<KeyValue> keyValueList = kvPair.getSecond();
+ keyValueList = preUpdateProcessor.preUpsert(kvPair.getFirst(), keyValueList);
+ byte[] first = kvPair.getFirst();
+ for (String tableName : tableNames) {
+ if (Bytes.compareTo(Bytes.toBytes(tableName), first) != 0) {
+ // skip edits for other tables
+ continue;
+ }
+ for (KeyValue kv : keyValueList) {
+ ImmutableBytesWritable outputKey = new ImmutableBytesWritable();
+ outputKey.set(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength());
+ context.write(new TableRowkeyPair(tableName, outputKey), kv);
+ }
+ }
+ }
+ conn.rollback();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ protected void cleanup(Context context) throws IOException, InterruptedException {
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Write the list of to-import columns to a job configuration.
+ *
+ * @param conf configuration to be written to
+ * @param columnInfoList list of ColumnInfo objects to be configured for import
+ */
+ @VisibleForTesting
+ static void configureColumnInfoList(Configuration conf, List<ColumnInfo> columnInfoList) {
+ conf.set(COLUMN_INFO_CONFKEY, Joiner.on("|").useForNull("").join(columnInfoList));
+ }
+
+ /**
+ * Build the list of ColumnInfos for the import based on information in the configuration.
+ */
+ @VisibleForTesting
+ static List<ColumnInfo> buildColumnInfoList(Configuration conf) {
+
+ return Lists.newArrayList(
+ Iterables.transform(
+ Splitter.on("|").split(conf.get(COLUMN_INFO_CONFKEY)),
+ new Function<String, ColumnInfo>() {
+ @Nullable
+ @Override
+ public ColumnInfo apply(@Nullable String input) {
+ if (input == null || input.isEmpty()) {
+ // An empty string represents a null that was passed in to
+ // the configuration, which corresponds to an input column
+ // which is to be skipped
+ return null;
+ }
+ return ColumnInfo.fromString(input);
+ }
+ }));
+ }
+
+ /**
+ * Listener that logs successful upserts and errors to job counters.
+ */
+ @VisibleForTesting
+ static class MapperUpsertListener<T> implements UpsertExecutor.UpsertListener<T> {
+
+ private final Mapper<LongWritable, Text, TableRowkeyPair, KeyValue>.Context context;
+ private final boolean ignoreRecordErrors;
+
+ private MapperUpsertListener(
+ Mapper<LongWritable, Text, TableRowkeyPair, KeyValue>.Context context,
+ boolean ignoreRecordErrors) {
+ this.context = context;
+ this.ignoreRecordErrors = ignoreRecordErrors;
+ }
+
+ @Override
+ public void upsertDone(long upsertCount) {
+ context.getCounter(COUNTER_GROUP_NAME, "Upserts Done").increment(1L);
+ }
+
+ @Override
+ public void errorOnRecord(T record, Throwable throwable) {
+ LOG.error("Error on record " + record, throwable);
+ context.getCounter(COUNTER_GROUP_NAME, "Errors on records").increment(1L);
+ if (!ignoreRecordErrors) {
+ throw Throwables.propagate(throwable);
+ }
+ }
+ }
+
+ /**
+ * A default implementation of {@code ImportPreUpsertKeyValueProcessor} that is used if no
+ * specific class is configured. This implementation simply passes through the KeyValue
+ * list that is passed in.
+ */
+ public static class DefaultImportPreUpsertKeyValueProcessor implements
+ ImportPreUpsertKeyValueProcessor {
+
+ @Override
+ public List<KeyValue> preUpsert(byte[] rowKey, List<KeyValue> keyValues) {
+ return keyValues;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueReducer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueReducer.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueReducer.java
new file mode 100644
index 0000000..5d00656
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueReducer.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce;
+
+import java.io.IOException;
+import java.util.TreeSet;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.mapreduce.KeyValueSortReducer;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.phoenix.mapreduce.bulkload.TableRowkeyPair;
+
+/**
+ * Reducer class for the bulkload jobs.
+ * Performs similar functionality to {@link KeyValueSortReducer}
+ */
+public class FormatToKeyValueReducer
+ extends Reducer<TableRowkeyPair,KeyValue,TableRowkeyPair,KeyValue> {
+
+ @Override
+ protected void reduce(TableRowkeyPair key, Iterable<KeyValue> values,
+ Reducer<TableRowkeyPair, KeyValue, TableRowkeyPair, KeyValue>.Context context)
+ throws IOException, InterruptedException {
+ TreeSet<KeyValue> map = new TreeSet<KeyValue>(KeyValue.COMPARATOR);
+ for (KeyValue kv: values) {
+ try {
+ map.add(kv.clone());
+ } catch (CloneNotSupportedException e) {
+ throw new java.io.IOException(e);
+ }
+ }
+ context.setStatus("Read " + map.getClass());
+ int index = 0;
+ for (KeyValue kv: map) {
+ context.write(key, kv);
+ if (++index % 100 == 0) context.setStatus("Wrote " + index);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/ImportPreUpsertKeyValueProcessor.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/ImportPreUpsertKeyValueProcessor.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/ImportPreUpsertKeyValueProcessor.java
index 62211f3..dff9ef2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/ImportPreUpsertKeyValueProcessor.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/ImportPreUpsertKeyValueProcessor.java
@@ -24,7 +24,8 @@ import java.util.List;
/**
* A listener hook to process KeyValues that are being written to HFiles for bulk import.
* Implementing this interface and configuring it via the {@link
- * CsvToKeyValueMapper#UPSERT_HOOK_CLASS_CONFKEY} configuration key.
+ * org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil#UPSERT_HOOK_CLASS_CONFKEY}
+ * configuration key.
* <p/>
* The intention of such a hook is to allow coproccessor-style operations to be peformed on
* data that is being bulk-loaded via MapReduce.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/JsonBulkLoadTool.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/JsonBulkLoadTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/JsonBulkLoadTool.java
new file mode 100644
index 0000000..1bea3f0
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/JsonBulkLoadTool.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce;
+
+import java.sql.SQLException;
+import java.util.List;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.phoenix.util.ColumnInfo;
+
+/**
+ * A tool for running MapReduce-based ingests of JSON data. Nested JSON data structures are not
+ * handled, though lists are converted into typed ARRAYS.
+ */
+public class JsonBulkLoadTool extends AbstractBulkLoadTool {
+
+ @Override
+ protected void configureOptions(CommandLine cmdLine, List<ColumnInfo> importColumns,
+ Configuration conf) throws SQLException {
+ // noop
+ }
+
+ @Override
+ protected void setupJob(Job job) {
+ // Allow overriding the job jar setting by using a -D system property at startup
+ if (job.getJar() == null) {
+ job.setJarByClass(JsonToKeyValueMapper.class);
+ }
+ job.setMapperClass(JsonToKeyValueMapper.class);
+ }
+
+ public static void main(String[] args) throws Exception {
+ ToolRunner.run(new JsonBulkLoadTool(), args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/JsonToKeyValueMapper.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/JsonToKeyValueMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/JsonToKeyValueMapper.java
new file mode 100644
index 0000000..5173a0e
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/JsonToKeyValueMapper.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.phoenix.util.ColumnInfo;
+import org.apache.phoenix.util.UpsertExecutor;
+import org.apache.phoenix.util.json.JsonUpsertExecutor;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+/**
+ * MapReduce mapper that converts JSON input lines into KeyValues that can be written to HFiles.
+ * <p/>
+ * KeyValues are produced by executing UPSERT statements on a Phoenix connection and then
+ * extracting the created KeyValues and rolling back the statement execution before it is
+ * committed to HBase.
+ */
+public class JsonToKeyValueMapper extends FormatToKeyValueMapper<Map<?, ?>> {
+
+ private LineParser<Map<?, ?>> lineParser;
+
+ @Override
+ protected LineParser<Map<?, ?>> getLineParser() {
+ return lineParser;
+ }
+
+ @Override
+ protected void setup(Context context) throws IOException, InterruptedException {
+ super.setup(context);
+ lineParser = new JsonLineParser();
+ }
+
+ @VisibleForTesting
+ @Override
+ protected UpsertExecutor<Map<?, ?>, ?> buildUpsertExecutor(Configuration conf) {
+ String tableName = conf.get(TABLE_NAME_CONFKEY);
+ Preconditions.checkNotNull(tableName, "table name is not configured");
+
+ List<ColumnInfo> columnInfoList = buildColumnInfoList(conf);
+
+ return new JsonUpsertExecutor(conn, tableName, columnInfoList, upsertListener);
+ }
+
+ @VisibleForTesting
+ static class JsonLineParser implements LineParser<Map<?, ?>> {
+ private final ObjectMapper mapper = new ObjectMapper();
+
+ @Override
+ public Map<?, ?> parse(String input) throws IOException {
+ return mapper.readValue(input, Map.class);
+ }
+ }
+}
[31/52] [abbrv] phoenix git commit: PHOENIX-2402 NPE when using
UPSERT SELECT with a char array (Julian Jaffe)
Posted by ma...@apache.org.
PHOENIX-2402 NPE when using UPSERT SELECT with a char array (Julian Jaffe)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/00ee9415
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/00ee9415
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/00ee9415
Branch: refs/heads/calcite
Commit: 00ee9415a95668c34e95b43003354fc898f6b4ea
Parents: 49be33e
Author: James Taylor <jt...@salesforce.com>
Authored: Tue Dec 1 18:56:35 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Tue Dec 1 18:56:35 2015 -0800
----------------------------------------------------------------------
.../apache/phoenix/end2end/UpsertSelectIT.java | 55 ++++++++++++++++++++
.../phoenix/compile/ProjectionCompiler.java | 4 +-
.../phoenix/expression/CoerceExpression.java | 8 ++-
3 files changed, 64 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/00ee9415/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
index aca1602..689562a 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
@@ -1318,6 +1318,61 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT {
}
}
+ @Test
+ public void testUpsertSelectWithFixedWidthNullByteSizeArray() throws Exception {
+ long ts = nextTimestamp();
+ Properties props = new Properties();
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts));
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement().execute(
+ "create table t1 (id bigint not null primary key, ca char(3)[])");
+ conn.close();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement().execute("upsert into t1 values (1, ARRAY['aaa', 'bbb'])");
+ conn.commit();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 15));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement().execute(
+ "upsert into t1(id, ca) select id, ARRAY['ccc', 'ddd'] from t1 WHERE id = 1");
+ conn.commit();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 20));
+ conn = DriverManager.getConnection(getUrl(), props);
+ ResultSet rs = conn.createStatement().executeQuery("select * from t1");
+
+ assertTrue(rs.next());
+ assertEquals(1, rs.getLong(1));
+ assertEquals("['ccc', 'ddd']", rs.getArray(2).toString());
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 25));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement().execute(
+ "create table t2 (id bigint not null primary key, ba binary(4)[])");
+ conn.close();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement().execute("upsert into t2 values (2, ARRAY[1, 27])");
+ conn.commit();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 35));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement().execute(
+ "upsert into t2(id, ba) select id, ARRAY[54, 1024] from t2 WHERE id = 2");
+ conn.commit();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 40));
+ conn = DriverManager.getConnection(getUrl(), props);
+ rs = conn.createStatement().executeQuery("select * from t2");
+
+ assertTrue(rs.next());
+ assertEquals(2, rs.getLong(1));
+ assertEquals("[[128,0,0,54], [128,0,4,0]]", rs.getArray(2).toString());
+ }
+
private static Connection getConnection(long ts) throws SQLException {
Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/00ee9415/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
index e477009..7cc2e66 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
@@ -78,12 +78,12 @@ import org.apache.phoenix.schema.PColumnFamily;
import org.apache.phoenix.schema.PDatum;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PTable;
-import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTable.ViewType;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.RowKeySchema;
+import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.ValueBitSet;
import org.apache.phoenix.schema.tuple.Tuple;
@@ -323,7 +323,7 @@ public class ProjectionCompiler {
if (expression.getDataType() != null && !expression.getDataType().isCastableTo(targetType)) {
throw new ArgumentTypeMismatchException(targetType, expression.getDataType(), "column: " + targetColumn);
}
- expression = CoerceExpression.create(expression, targetType);
+ expression = CoerceExpression.create(expression, targetType, targetColumn.getSortOrder(), targetColumn.getMaxLength());
}
}
return expression;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/00ee9415/phoenix-core/src/main/java/org/apache/phoenix/expression/CoerceExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/CoerceExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/CoerceExpression.java
index fb6a825..c31cb0a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/CoerceExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/CoerceExpression.java
@@ -50,8 +50,14 @@ public class CoerceExpression extends BaseSingleExpression {
return new CoerceExpression(expression, toType);
}
+ public static Expression create(Expression expression, PDataType toType, SortOrder toSortOrder, Integer maxLength) throws SQLException {
+ return create(expression, toType, toSortOrder, maxLength, true);
+ }
+
public static Expression create(Expression expression, PDataType toType, SortOrder toSortOrder, Integer maxLength, boolean rowKeyOrderOptimizable) throws SQLException {
- if (toType == expression.getDataType() && toSortOrder == expression.getSortOrder()) {
+ if ( toType == expression.getDataType() &&
+ toSortOrder == expression.getSortOrder() &&
+ (maxLength == null || maxLength.equals(expression.getMaxLength())) ) {
return expression;
}
return new CoerceExpression(expression, toType, toSortOrder, maxLength, rowKeyOrderOptimizable);
[04/52] [abbrv] phoenix git commit: PHOENIX-1674 Snapshot isolation
transaction support through Tephra (James Taylor, Thomas D'Silva)
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
new file mode 100644
index 0000000..909d59b
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.index;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellScanner;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
+import org.apache.phoenix.compile.ScanRanges;
+import org.apache.phoenix.hbase.index.MultiMutation;
+import org.apache.phoenix.hbase.index.ValueGetter;
+import org.apache.phoenix.hbase.index.covered.IndexUpdate;
+import org.apache.phoenix.hbase.index.covered.TableState;
+import org.apache.phoenix.hbase.index.covered.update.ColumnReference;
+import org.apache.phoenix.hbase.index.covered.update.ColumnTracker;
+import org.apache.phoenix.hbase.index.covered.update.IndexedColumnGroup;
+import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.hbase.index.write.IndexWriter;
+import org.apache.phoenix.query.KeyRange;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.types.PVarbinary;
+import org.apache.phoenix.trace.TracingUtils;
+import org.apache.phoenix.trace.util.NullSpan;
+import org.apache.phoenix.util.ScanUtil;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.ServerUtil;
+
+import co.cask.tephra.Transaction;
+import co.cask.tephra.Transaction.VisibilityLevel;
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.hbase11.TransactionAwareHTable;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.primitives.Longs;
+
+/**
+ * Do all the work of managing index updates for a transactional table from a single coprocessor. Since the transaction
+ * manager essentially time orders writes through conflict detection, the logic to maintain a secondary index is quite a
+ * bit simpler than the non transactional case. For example, there's no need to muck with the WAL, as failure scenarios
+ * are handled by aborting the transaction.
+ */
+public class PhoenixTransactionalIndexer extends BaseRegionObserver {
+
+ private static final Log LOG = LogFactory.getLog(PhoenixTransactionalIndexer.class);
+
+ private PhoenixIndexCodec codec;
+ private IndexWriter writer;
+ private boolean stopped;
+
+ @Override
+ public void start(CoprocessorEnvironment e) throws IOException {
+ final RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment)e;
+ String serverName = env.getRegionServerServices().getServerName().getServerName();
+ codec = new PhoenixIndexCodec();
+ codec.initialize(env);
+
+ // setup the actual index writer
+ this.writer = new IndexWriter(env, serverName + "-tx-index-writer");
+ }
+
+ @Override
+ public void stop(CoprocessorEnvironment e) throws IOException {
+ if (this.stopped) { return; }
+ this.stopped = true;
+ String msg = "TxIndexer is being stopped";
+ this.writer.stop(msg);
+ }
+
+ private static Iterator<Mutation> getMutationIterator(final MiniBatchOperationInProgress<Mutation> miniBatchOp) {
+ return new Iterator<Mutation>() {
+ private int i = 0;
+
+ @Override
+ public boolean hasNext() {
+ return i < miniBatchOp.size();
+ }
+
+ @Override
+ public Mutation next() {
+ return miniBatchOp.getOperation(i++);
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+ @Override
+ public void preBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c,
+ MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
+
+ Mutation m = miniBatchOp.getOperation(0);
+ if (!codec.isEnabled(m)) {
+ super.preBatchMutate(c, miniBatchOp);
+ return;
+ }
+
+ Map<String,byte[]> updateAttributes = m.getAttributesMap();
+ PhoenixIndexMetaData indexMetaData = new PhoenixIndexMetaData(c.getEnvironment(),updateAttributes);
+ byte[] txRollbackAttribute = m.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY);
+ Collection<Pair<Mutation, byte[]>> indexUpdates = null;
+ // get the current span, or just use a null-span to avoid a bunch of if statements
+ try (TraceScope scope = Trace.startSpan("Starting to build index updates")) {
+ Span current = scope.getSpan();
+ if (current == null) {
+ current = NullSpan.INSTANCE;
+ }
+
+ // get the index updates for all elements in this batch
+ indexUpdates = getIndexUpdates(c.getEnvironment(), indexMetaData, getMutationIterator(miniBatchOp), txRollbackAttribute);
+
+ current.addTimelineAnnotation("Built index updates, doing preStep");
+ TracingUtils.addAnnotation(current, "index update count", indexUpdates.size());
+
+ // no index updates, so we are done
+ if (!indexUpdates.isEmpty()) {
+ this.writer.write(indexUpdates);
+ }
+ } catch (Throwable t) {
+ String msg = "Failed to update index with entries:" + indexUpdates;
+ LOG.error(msg, t);
+ ServerUtil.throwIOException(msg, t);
+ }
+ }
+
+ private Collection<Pair<Mutation, byte[]>> getIndexUpdates(RegionCoprocessorEnvironment env, PhoenixIndexMetaData indexMetaData, Iterator<Mutation> mutationIterator, byte[] txRollbackAttribute) throws IOException {
+ ResultScanner currentScanner = null;
+ TransactionAwareHTable txTable = null;
+ // Collect up all mutations in batch
+ Map<ImmutableBytesPtr, MultiMutation> mutations =
+ new HashMap<ImmutableBytesPtr, MultiMutation>();
+ while(mutationIterator.hasNext()) {
+ Mutation m = mutationIterator.next();
+ // add the mutation to the batch set
+ ImmutableBytesPtr row = new ImmutableBytesPtr(m.getRow());
+ MultiMutation stored = mutations.get(row);
+ // we haven't seen this row before, so add it
+ if (stored == null) {
+ stored = new MultiMutation(row);
+ mutations.put(row, stored);
+ }
+ stored.addAll(m);
+ }
+
+ // Collect the set of mutable ColumnReferences so that we can first
+ // run a scan to get the current state. We'll need this to delete
+ // the existing index rows.
+ Transaction tx = indexMetaData.getTransaction();
+ assert(tx != null);
+ List<IndexMaintainer> indexMaintainers = indexMetaData.getIndexMaintainers();
+ Set<ColumnReference> mutableColumns = Sets.newHashSetWithExpectedSize(indexMaintainers.size() * 10);
+ for (IndexMaintainer indexMaintainer : indexMaintainers) {
+ // For transactional tables, we use an index maintainer
+ // to aid in rollback if there's a KeyValue column in the index. The alternative would be
+ // to hold on to all uncommitted index row keys (even ones already sent to HBase) on the
+ // client side.
+ mutableColumns.addAll(indexMaintainer.getAllColumns());
+ }
+
+ boolean isRollback = txRollbackAttribute!=null;
+ Collection<Pair<Mutation, byte[]>> indexUpdates = new ArrayList<Pair<Mutation, byte[]>>(mutations.size() * 2 * indexMaintainers.size());
+ try {
+ if (!mutableColumns.isEmpty()) {
+ List<KeyRange> keys = Lists.newArrayListWithExpectedSize(mutations.size());
+ for (ImmutableBytesPtr ptr : mutations.keySet()) {
+ keys.add(PVarbinary.INSTANCE.getKeyRange(ptr.copyBytesIfNecessary()));
+ }
+ Scan scan = new Scan();
+ // Project all mutable columns
+ for (ColumnReference ref : mutableColumns) {
+ scan.addColumn(ref.getFamily(), ref.getQualifier());
+ }
+ // Project empty key value column
+ scan.addColumn(indexMaintainers.get(0).getDataEmptyKeyValueCF(), QueryConstants.EMPTY_COLUMN_BYTES);
+ ScanRanges scanRanges = ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, KeyRange.EVERYTHING_RANGE, null, true, -1);
+ scanRanges.initializeScan(scan);
+ scan.setFilter(scanRanges.getSkipScanFilter());
+ TableName tableName = env.getRegion().getRegionInfo().getTable();
+ HTableInterface htable = env.getTable(tableName);
+ txTable = new TransactionAwareHTable(htable);
+ txTable.startTx(tx);
+ // For rollback, we need to see all versions, including
+ // the last committed version as there may be multiple
+ // checkpointed versions.
+ if (isRollback) {
+ tx.setVisibility(VisibilityLevel.SNAPSHOT_ALL);
+ }
+ currentScanner = txTable.getScanner(scan);
+ }
+ if (isRollback) {
+ processRollback(env, indexMetaData, txRollbackAttribute, currentScanner, mutations, tx, mutableColumns, indexUpdates);
+ } else {
+ processMutation(env, indexMetaData, txRollbackAttribute, currentScanner, mutations, tx, mutableColumns, indexUpdates);
+ }
+ } finally {
+ if (txTable != null) txTable.close();
+ }
+
+ return indexUpdates;
+ }
+
+ private void processMutation(RegionCoprocessorEnvironment env,
+ PhoenixIndexMetaData indexMetaData, byte[] txRollbackAttribute,
+ ResultScanner scanner,
+ Map<ImmutableBytesPtr, MultiMutation> mutations, Transaction tx,
+ Set<ColumnReference> mutableColumns,
+ Collection<Pair<Mutation, byte[]>> indexUpdates) throws IOException {
+ if (scanner != null) {
+ Result result;
+ ColumnReference emptyColRef = new ColumnReference(indexMetaData.getIndexMaintainers().get(0).getDataEmptyKeyValueCF(), QueryConstants.EMPTY_COLUMN_BYTES);
+ // Process existing data table rows by removing the old index row and adding the new index row
+ while ((result = scanner.next()) != null) {
+ Mutation m = mutations.remove(new ImmutableBytesPtr(result.getRow()));
+ TxTableState state = new TxTableState(env, mutableColumns, indexMetaData.getAttributes(), tx.getWritePointer(), m, emptyColRef, result);
+ generateDeletes(indexMetaData, indexUpdates, txRollbackAttribute, state);
+ generatePuts(indexMetaData, indexUpdates, state);
+ }
+ }
+ // Process new data table by adding new index rows
+ for (Mutation m : mutations.values()) {
+ TxTableState state = new TxTableState(env, mutableColumns, indexMetaData.getAttributes(), tx.getWritePointer(), m);
+ generatePuts(indexMetaData, indexUpdates, state);
+ }
+ }
+
+ private void processRollback(RegionCoprocessorEnvironment env,
+ PhoenixIndexMetaData indexMetaData, byte[] txRollbackAttribute,
+ ResultScanner scanner,
+ Map<ImmutableBytesPtr, MultiMutation> mutations, Transaction tx,
+ Set<ColumnReference> mutableColumns,
+ Collection<Pair<Mutation, byte[]>> indexUpdates) throws IOException {
+ if (scanner != null) {
+ Result result;
+ // Loop through last committed row state plus all new rows associated with current transaction
+ // to generate point delete markers for all index rows that were added. We don't have Tephra
+ // manage index rows in change sets because we don't want to be hit with the additional
+ // memory hit and do not need to do conflict detection on index rows.
+ ColumnReference emptyColRef = new ColumnReference(indexMetaData.getIndexMaintainers().get(0).getDataEmptyKeyValueCF(), QueryConstants.EMPTY_COLUMN_BYTES);
+ while ((result = scanner.next()) != null) {
+ Mutation m = mutations.remove(new ImmutableBytesPtr(result.getRow()));
+ // Sort by timestamp, type, cf, cq so we can process in time batches from oldest to newest
+ // (as if we're "replaying" them in time order).
+ List<Cell> cells = result.listCells();
+ Collections.sort(cells, new Comparator<Cell>() {
+
+ @Override
+ public int compare(Cell o1, Cell o2) {
+ int c = Longs.compare(o1.getTimestamp(), o2.getTimestamp());
+ if (c != 0) return c;
+ c = o1.getTypeByte() - o2.getTypeByte();
+ if (c != 0) return c;
+ c = Bytes.compareTo(o1.getFamilyArray(), o1.getFamilyOffset(), o1.getFamilyLength(), o1.getFamilyArray(), o1.getFamilyOffset(), o1.getFamilyLength());
+ if (c != 0) return c;
+ return Bytes.compareTo(o1.getQualifierArray(), o1.getQualifierOffset(), o1.getQualifierLength(), o1.getQualifierArray(), o1.getQualifierOffset(), o1.getQualifierLength());
+ }
+
+ });
+ int i = 0;
+ int nCells = cells.size();
+ Result oldResult = null, newResult;
+ long readPtr = tx.getReadPointer();
+ do {
+ boolean hasPuts = false;
+ LinkedList<Cell> singleTimeCells = Lists.newLinkedList();
+ long writePtr;
+ do {
+ Cell cell = cells.get(i);
+ hasPuts |= cell.getTypeByte() == KeyValue.Type.Put.getCode();
+ writePtr = cell.getTimestamp();
+ do {
+ // Add at the beginning of the list to match the expected HBase
+ // newest to oldest sort order (which TxTableState relies on
+ // with the Result.getLatestColumnValue() calls).
+ singleTimeCells.addFirst(cell);
+ } while (++i < nCells && cells.get(i).getTimestamp() == writePtr);
+ } while (i < nCells && cells.get(i).getTimestamp() <= readPtr);
+
+ // Generate point delete markers for the prior row deletion of the old index value.
+ // The write timestamp is the next timestamp, not the current timestamp,
+ // as the earliest cells are the current values for the row (and we don't
+ // want to delete the current row).
+ if (oldResult != null) {
+ TxTableState state = new TxTableState(env, mutableColumns, indexMetaData.getAttributes(), writePtr, m, emptyColRef, oldResult);
+ generateDeletes(indexMetaData, indexUpdates, txRollbackAttribute, state);
+ }
+ // Generate point delete markers for the new index value.
+ // If our time batch doesn't have Puts (i.e. we have only Deletes), then do not
+ // generate deletes. We would have generated the delete above based on the state
+ // of the previous row. The delete markers do not give us the state we need to
+ // delete.
+ if (hasPuts) {
+ newResult = Result.create(singleTimeCells);
+ // First row may represent the current state which we don't want to delete
+ if (writePtr > readPtr) {
+ TxTableState state = new TxTableState(env, mutableColumns, indexMetaData.getAttributes(), writePtr, m, emptyColRef, newResult);
+ generateDeletes(indexMetaData, indexUpdates, txRollbackAttribute, state);
+ }
+ oldResult = newResult;
+ } else {
+ oldResult = null;
+ }
+ } while (i < nCells);
+ }
+ }
+ }
+
+ private void generateDeletes(PhoenixIndexMetaData indexMetaData,
+ Collection<Pair<Mutation, byte[]>> indexUpdates,
+ byte[] attribValue, TxTableState state) throws IOException {
+ Iterable<IndexUpdate> deletes = codec.getIndexDeletes(state, indexMetaData);
+ for (IndexUpdate delete : deletes) {
+ if (delete.isValid()) {
+ delete.getUpdate().setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, attribValue);
+ indexUpdates.add(new Pair<Mutation, byte[]>(delete.getUpdate(),delete.getTableName()));
+ }
+ }
+ }
+
+ private boolean generatePuts(
+ PhoenixIndexMetaData indexMetaData,
+ Collection<Pair<Mutation, byte[]>> indexUpdates,
+ TxTableState state)
+ throws IOException {
+ state.applyMutation();
+ Iterable<IndexUpdate> puts = codec.getIndexUpserts(state, indexMetaData);
+ boolean validPut = false;
+ for (IndexUpdate put : puts) {
+ if (put.isValid()) {
+ indexUpdates.add(new Pair<Mutation, byte[]>(put.getUpdate(),put.getTableName()));
+ validPut = true;
+ }
+ }
+ return validPut;
+ }
+
+
+ private static class TxTableState implements TableState {
+ private final Mutation mutation;
+ private final long currentTimestamp;
+ private final RegionCoprocessorEnvironment env;
+ private final Map<String, byte[]> attributes;
+ private final List<KeyValue> pendingUpdates;
+ private final Set<ColumnReference> indexedColumns;
+ private final Map<ColumnReference, ImmutableBytesWritable> valueMap;
+
+ private TxTableState(RegionCoprocessorEnvironment env, Set<ColumnReference> indexedColumns, Map<String, byte[]> attributes, long currentTimestamp, Mutation mutation) {
+ this.env = env;
+ this.currentTimestamp = currentTimestamp;
+ this.indexedColumns = indexedColumns;
+ this.attributes = attributes;
+ this.mutation = mutation;
+ int estimatedSize = indexedColumns.size();
+ this.valueMap = Maps.newHashMapWithExpectedSize(estimatedSize);
+ this.pendingUpdates = Lists.newArrayListWithExpectedSize(estimatedSize);
+ try {
+ CellScanner scanner = mutation.cellScanner();
+ while (scanner.advance()) {
+ Cell cell = scanner.current();
+ pendingUpdates.add(KeyValueUtil.ensureKeyValue(cell));
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e); // Impossible
+ }
+ }
+
+ public TxTableState(RegionCoprocessorEnvironment env, Set<ColumnReference> indexedColumns, Map<String, byte[]> attributes, long currentTimestamp, Mutation m, ColumnReference emptyColRef, Result r) {
+ this(env, indexedColumns, attributes, currentTimestamp, m);
+
+ for (ColumnReference ref : indexedColumns) {
+ Cell cell = r.getColumnLatestCell(ref.getFamily(), ref.getQualifier());
+ if (cell != null) {
+ ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+ ptr.set(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ valueMap.put(ref, ptr);
+ }
+ }
+ }
+
+ @Override
+ public RegionCoprocessorEnvironment getEnvironment() {
+ return env;
+ }
+
+ @Override
+ public long getCurrentTimestamp() {
+ return currentTimestamp;
+ }
+
+ @Override
+ public Map<String, byte[]> getUpdateAttributes() {
+ return attributes;
+ }
+
+ @Override
+ public byte[] getCurrentRowKey() {
+ return mutation.getRow();
+ }
+
+ @Override
+ public List<? extends IndexedColumnGroup> getIndexColumnHints() {
+ return Collections.emptyList();
+ }
+
+ private void applyMutation() {
+ for (Cell cell : pendingUpdates) {
+ if (cell.getTypeByte() == KeyValue.Type.Delete.getCode() || cell.getTypeByte() == KeyValue.Type.DeleteColumn.getCode()) {
+ ColumnReference ref = new ColumnReference(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(), cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
+ valueMap.remove(ref);
+ } else if (cell.getTypeByte() == KeyValue.Type.DeleteFamily.getCode() || cell.getTypeByte() == KeyValue.Type.DeleteFamilyVersion.getCode()) {
+ for (ColumnReference ref : indexedColumns) {
+ if (ref.matchesFamily(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength())) {
+ valueMap.remove(ref);
+ }
+ }
+ } else if (cell.getTypeByte() == KeyValue.Type.Put.getCode()){
+ ColumnReference ref = new ColumnReference(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(), cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
+ if (indexedColumns.contains(ref)) {
+ ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+ ptr.set(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ valueMap.put(ref, ptr);
+ }
+ } else {
+ throw new IllegalStateException("Unexpected mutation type for " + cell);
+ }
+ }
+ }
+
+ @Override
+ public Collection<KeyValue> getPendingUpdate() {
+ return pendingUpdates;
+ }
+
+ @Override
+ public Pair<ValueGetter, IndexUpdate> getIndexUpdateState(Collection<? extends ColumnReference> indexedColumns)
+ throws IOException {
+ // TODO: creating these objects over and over again is wasteful
+ ColumnTracker tracker = new ColumnTracker(indexedColumns);
+ ValueGetter getter = new ValueGetter() {
+
+ @Override
+ public ImmutableBytesWritable getLatestValue(ColumnReference ref) throws IOException {
+ return valueMap.get(ref);
+ }
+
+ @Override
+ public byte[] getRowKey() {
+ return mutation.getRow();
+ }
+
+ };
+ Pair<ValueGetter, IndexUpdate> pair = new Pair<ValueGetter, IndexUpdate>(getter, new IndexUpdate(tracker));
+ return pair;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 57e84f0..c18233e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -55,6 +55,7 @@ import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
import org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.exception.SQLExceptionInfo;
+import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.filter.ColumnProjectionFilter;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.parse.FilterableStatement;
@@ -104,6 +105,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
private final byte[] physicalTableName;
private final QueryPlan plan;
protected final String scanId;
+ protected final MutationState mutationState;
private final ParallelScanGrouper scanGrouper;
// TODO: too much nesting here - breakup into new classes.
private final List<List<List<Pair<Scan,Future<PeekingResultIterator>>>>> allFutures;
@@ -202,6 +204,9 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
this.plan = plan;
this.scanGrouper = scanGrouper;
StatementContext context = plan.getContext();
+ // Clone MutationState as the one on the connection will change if auto commit is on
+ // yet we need the original one with the original transaction from TableResultIterator.
+ this.mutationState = new MutationState(context.getConnection().getMutationState());
TableRef tableRef = plan.getTableRef();
PTable table = tableRef.getTable();
physicalTableName = table.getPhysicalName().getBytes();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java
index f272e55..ef8c4a6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.compile.StatementContext;
+import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.TableRef;
@@ -51,38 +52,44 @@ public class ChunkedResultIterator implements PeekingResultIterator {
private ImmutableBytesWritable lastKey = new ImmutableBytesWritable();
private final StatementContext context;
private final TableRef tableRef;
- private Scan scan;
private final long chunkSize;
+ private final MutationState mutationState;
+ private Scan scan;
private PeekingResultIterator resultIterator;
public static class ChunkedResultIteratorFactory implements ParallelIteratorFactory {
private final ParallelIteratorFactory delegateFactory;
private final TableRef tableRef;
+ private final MutationState mutationState;
public ChunkedResultIteratorFactory(ParallelIteratorFactory
- delegateFactory, TableRef tableRef) {
+ delegateFactory, MutationState mutationState, TableRef tableRef) {
this.delegateFactory = delegateFactory;
this.tableRef = tableRef;
+ // Clone MutationState, as the one on the connection may change if auto commit is on
+ // while we need a handle to the original one (for it's transaction state).
+ this.mutationState = new MutationState(mutationState);
}
@Override
public PeekingResultIterator newIterator(StatementContext context, ResultIterator scanner, Scan scan, String tableName) throws SQLException {
if (logger.isDebugEnabled()) logger.debug(LogUtil.addCustomAnnotations("ChunkedResultIteratorFactory.newIterator over " + tableRef.getTable().getName().getString() + " with " + scan, ScanUtil.getCustomAnnotations(scan)));
- return new ChunkedResultIterator(delegateFactory, context, tableRef, scan,
- context.getConnection().getQueryServices().getProps().getLong(
- QueryServices.SCAN_RESULT_CHUNK_SIZE,
- QueryServicesOptions.DEFAULT_SCAN_RESULT_CHUNK_SIZE), scanner);
+ return new ChunkedResultIterator(delegateFactory, mutationState, context, tableRef, scan,
+ mutationState.getConnection().getQueryServices().getProps().getLong(
+ QueryServices.SCAN_RESULT_CHUNK_SIZE,
+ QueryServicesOptions.DEFAULT_SCAN_RESULT_CHUNK_SIZE), scanner);
}
}
- private ChunkedResultIterator(ParallelIteratorFactory delegateIteratorFactory,
- StatementContext context, TableRef tableRef, Scan scan, long chunkSize, ResultIterator scanner) throws SQLException {
+ private ChunkedResultIterator(ParallelIteratorFactory delegateIteratorFactory, MutationState mutationState,
+ StatementContext context, TableRef tableRef, Scan scan, long chunkSize, ResultIterator scanner) throws SQLException {
this.delegateIteratorFactory = delegateIteratorFactory;
this.context = context;
this.tableRef = tableRef;
this.scan = scan;
this.chunkSize = chunkSize;
+ this.mutationState = mutationState;
// Instantiate single chunk iterator and the delegate iterator in constructor
// to get parallel scans kicked off in separate threads. If we delay this,
// we'll get serialized behavior (see PHOENIX-
@@ -120,7 +127,7 @@ public class ChunkedResultIterator implements PeekingResultIterator {
if (logger.isDebugEnabled()) logger.debug(LogUtil.addCustomAnnotations("Get next chunked result iterator over " + tableRef.getTable().getName().getString() + " with " + scan, ScanUtil.getCustomAnnotations(scan)));
String tableName = tableRef.getTable().getPhysicalName().getString();
ResultIterator singleChunkResultIterator = new SingleChunkResultIterator(
- new TableResultIterator(context, tableRef, scan, context.getReadMetricsQueue().allotMetric(SCAN_BYTES, tableName)), chunkSize);
+ new TableResultIterator(mutationState, tableRef, scan, context.getReadMetricsQueue().allotMetric(SCAN_BYTES, tableName)), chunkSize);
resultIterator = delegateIteratorFactory.newIterator(context, singleChunkResultIterator, scan, tableName);
}
return resultIterator;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
index 87f8335..265831c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
@@ -102,7 +102,7 @@ public class ParallelIterators extends BaseResultIterators {
@Override
public PeekingResultIterator call() throws Exception {
long startTime = System.currentTimeMillis();
- ResultIterator scanner = new TableResultIterator(context, tableRef, scan, scanMetrics);
+ ResultIterator scanner = new TableResultIterator(mutationState, tableRef, scan, scanMetrics);
if (logger.isDebugEnabled()) {
logger.debug(LogUtil.addCustomAnnotations("Id: " + scanId + ", Time: " + (System.currentTimeMillis() - startTime) + "ms, Scan: " + scan, ScanUtil.getCustomAnnotations(scan)));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
index fa18c83..e0936b2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
@@ -80,7 +80,7 @@ public class SerialIterators extends BaseResultIterators {
public PeekingResultIterator call() throws Exception {
List<PeekingResultIterator> concatIterators = Lists.newArrayListWithExpectedSize(scans.size());
for (final Scan scan : scans) {
- ResultIterator scanner = new TableResultIterator(context, tableRef, scan, context.getReadMetricsQueue().allotMetric(SCAN_BYTES, tableName), ScannerCreation.DELAYED);
+ ResultIterator scanner = new TableResultIterator(mutationState, tableRef, scan, context.getReadMetricsQueue().allotMetric(SCAN_BYTES, tableName), ScannerCreation.DELAYED);
concatIterators.add(iteratorFactory.newIterator(context, scanner, scan, tableName));
}
PeekingResultIterator concatIterator = ConcatResultIterator.newIterator(concatIterators);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java
index 6f040d1..7d07a4a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java
@@ -22,8 +22,11 @@ import java.sql.SQLException;
import java.util.List;
import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.phoenix.compile.StatementContext;
+import org.apache.phoenix.execute.MutationState;
+import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.monitoring.CombinableMetric;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.tuple.Tuple;
@@ -39,7 +42,7 @@ import org.apache.phoenix.util.ServerUtil;
*
* @since 0.1
*/
-public class TableResultIterator extends ExplainTable implements ResultIterator {
+public class TableResultIterator implements ResultIterator {
public enum ScannerCreation {IMMEDIATE, DELAYED};
private final Scan scan;
@@ -47,8 +50,8 @@ public class TableResultIterator extends ExplainTable implements ResultIterator
private volatile ResultIterator delegate;
private final CombinableMetric scanMetrics;
- public TableResultIterator(StatementContext context, TableRef tableRef, CombinableMetric scanMetrics) throws SQLException {
- this(context, tableRef, context.getScan(), scanMetrics);
+ public TableResultIterator(MutationState mutationState, Scan scan, TableRef tableRef, CombinableMetric scanMetrics) throws SQLException {
+ this(mutationState, tableRef, scan, scanMetrics);
}
/*
@@ -75,15 +78,15 @@ public class TableResultIterator extends ExplainTable implements ResultIterator
return delegate;
}
- public TableResultIterator(StatementContext context, TableRef tableRef, Scan scan, CombinableMetric scanMetrics) throws SQLException {
- this(context, tableRef, scan, scanMetrics, ScannerCreation.IMMEDIATE);
+ public TableResultIterator(MutationState mutationState, TableRef tableRef, Scan scan, CombinableMetric scanMetrics) throws SQLException {
+ this(mutationState, tableRef, scan, scanMetrics, ScannerCreation.IMMEDIATE);
}
- public TableResultIterator(StatementContext context, TableRef tableRef, Scan scan, CombinableMetric scanMetrics, ScannerCreation creationMode) throws SQLException {
- super(context, tableRef);
+ public TableResultIterator(MutationState mutationState, TableRef tableRef, Scan scan, CombinableMetric scanMetrics, ScannerCreation creationMode) throws SQLException {
this.scan = scan;
this.scanMetrics = scanMetrics;
- htable = context.getConnection().getQueryServices().getTable(tableRef.getTable().getPhysicalName().getBytes());
+ PTable table = tableRef.getTable();
+ htable = mutationState.getHTable(table);
if (creationMode == ScannerCreation.IMMEDIATE) {
getDelegate(false);
}
@@ -109,8 +112,11 @@ public class TableResultIterator extends ExplainTable implements ResultIterator
@Override
public void explain(List<String> planSteps) {
- StringBuilder buf = new StringBuilder();
- explain(buf.toString(),planSteps);
+ try {
+ getDelegate(false).explain(planSteps);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
index 6a142e5..e864bfe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
@@ -79,6 +79,7 @@ import org.apache.phoenix.schema.PMetaData.Pruner;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.PTableRef;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.schema.types.PArrayDataType;
@@ -158,14 +159,18 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
}
public PhoenixConnection(PhoenixConnection connection, boolean isDescRowKeyOrderUpgrade) throws SQLException {
- this(connection.getQueryServices(), connection.getURL(), connection.getClientInfo(), connection.getMetaDataCache(), isDescRowKeyOrderUpgrade);
+ this(connection.getQueryServices(), connection.getURL(), connection.getClientInfo(), connection.metaData, connection.getMutationState(), isDescRowKeyOrderUpgrade);
this.isAutoCommit = connection.isAutoCommit;
this.sampler = connection.sampler;
this.statementExecutionCounter = connection.statementExecutionCounter;
}
public PhoenixConnection(PhoenixConnection connection) throws SQLException {
- this(connection, connection.isDescVarLengthRowKeyUpgrade);
+ this(connection, connection.isDescVarLengthRowKeyUpgrade());
+ }
+
+ public PhoenixConnection(PhoenixConnection connection, MutationState mutationState) throws SQLException {
+ this(connection.getQueryServices(), connection.getURL(), connection.getClientInfo(), connection.getMetaDataCache(), mutationState, connection.isDescVarLengthRowKeyUpgrade());
}
public PhoenixConnection(PhoenixConnection connection, long scn) throws SQLException {
@@ -173,17 +178,21 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
}
public PhoenixConnection(ConnectionQueryServices services, PhoenixConnection connection, long scn) throws SQLException {
- this(services, connection.getURL(), newPropsWithSCN(scn,connection.getClientInfo()), connection.getMetaDataCache(), connection.isDescVarLengthRowKeyUpgrade());
+ this(services, connection.getURL(), newPropsWithSCN(scn,connection.getClientInfo()), connection.metaData, connection.getMutationState(), connection.isDescVarLengthRowKeyUpgrade());
this.isAutoCommit = connection.isAutoCommit;
this.sampler = connection.sampler;
this.statementExecutionCounter = connection.statementExecutionCounter;
}
public PhoenixConnection(ConnectionQueryServices services, String url, Properties info, PMetaData metaData) throws SQLException {
- this(services, url, info, metaData, false);
+ this(services, url, info, metaData, null, false);
}
-
- public PhoenixConnection(ConnectionQueryServices services, String url, Properties info, PMetaData metaData, boolean isDescVarLengthRowKeyUpgrade) throws SQLException {
+
+ public PhoenixConnection(PhoenixConnection connection, ConnectionQueryServices services, Properties info) throws SQLException {
+ this(services, connection.url, info, connection.metaData, null, connection.isDescVarLengthRowKeyUpgrade());
+ }
+
+ public PhoenixConnection(ConnectionQueryServices services, String url, Properties info, PMetaData metaData, MutationState mutationState, boolean isDescVarLengthRowKeyUpgrade) throws SQLException {
this.url = url;
this.isDescVarLengthRowKeyUpgrade = isDescVarLengthRowKeyUpgrade;
// Copy so client cannot change
@@ -255,7 +264,7 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
}
};
this.isRequestLevelMetricsEnabled = JDBCUtil.isCollectingRequestLevelMetricsEnabled(url, info, this.services.getProps());
- this.mutationState = newMutationState(maxSize);
+ this.mutationState = mutationState == null ? newMutationState(maxSize) : new MutationState(mutationState);
this.metaData = metaData.pruneTables(pruner);
this.metaData = metaData.pruneFunctions(pruner);
this.services.addConnection(this);
@@ -408,12 +417,16 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
return mutateBatchSize;
}
+ public PMetaData getMetaDataCache() {
+ return metaData;
+ }
+
public PTable getTable(PTableKey key) throws TableNotFoundException {
- return metaData.getTable(key);
+ return metaData.getTableRef(key).getTable();
}
- public PMetaData getMetaDataCache() {
- return metaData;
+ public PTableRef getTableRef(PTableKey key) throws TableNotFoundException {
+ return metaData.getTableRef(key);
}
protected MutationState newMutationState(int maxSize) {
@@ -450,7 +463,9 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
// from modifying this list.
this.statements = Lists.newArrayList();
try {
- mutationState.rollback(this);
+ mutationState.rollback();
+ } catch (SQLException e) {
+ // ignore any exceptions while rolling back
} finally {
try {
SQLCloseables.closeAll(statements);
@@ -693,7 +708,13 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
@Override
public void rollback() throws SQLException {
- mutationState.rollback(this);
+ CallRunner.run(new CallRunner.CallableThrowable<Void, SQLException>() {
+ @Override
+ public Void call() throws SQLException {
+ mutationState.rollback();
+ return null;
+ }
+ }, Tracing.withTracing(this, "rolling back"));
statementExecutionCounter = 0;
}
@@ -806,16 +827,20 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
// TODO Auto-generated method stub
return 0;
}
-
+
@Override
- public PMetaData addTable(PTable table) throws SQLException {
- // TODO: since a connection is only used by one thread at a time,
- // we could modify this metadata in place since it's not shared.
- if (scn == null || scn > table.getTimeStamp()) {
- metaData = metaData.addTable(table);
- }
+ public PMetaData addTable(PTable table, long resolvedTime) throws SQLException {
+ metaData = metaData.addTable(table, resolvedTime);
//Cascade through to connectionQueryServices too
- getQueryServices().addTable(table);
+ getQueryServices().addTable(table, resolvedTime);
+ return metaData;
+ }
+
+ @Override
+ public PMetaData updateResolvedTimestamp(PTable table, long resolvedTime) throws SQLException {
+ metaData = metaData.updateResolvedTimestamp(table, resolvedTime);
+ //Cascade through to connectionQueryServices too
+ getQueryServices().updateResolvedTimestamp(table, resolvedTime);
return metaData;
}
@@ -832,11 +857,11 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
}
@Override
- public PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls)
+ public PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime)
throws SQLException {
- metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls);
+ metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime);
//Cascade through to connectionQueryServices too
- getQueryServices().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls);
+ getQueryServices().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime);
return metaData;
}
@@ -858,10 +883,10 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
@Override
public PMetaData removeColumn(PName tenantId, String tableName, List<PColumn> columnsToRemove, long tableTimeStamp,
- long tableSeqNum) throws SQLException {
- metaData = metaData.removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum);
+ long tableSeqNum, long resolvedTime) throws SQLException {
+ metaData = metaData.removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum, resolvedTime);
//Cascade through to connectionQueryServices too
- getQueryServices().removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum);
+ getQueryServices().removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum, resolvedTime);
return metaData;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index 3f6958c..d6d1230 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -301,6 +301,9 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData, org.apache.pho
private static final String TENANT_POS_SHIFT = "TENANT_POS_SHIFT";
private static final byte[] TENANT_POS_SHIFT_BYTES = Bytes.toBytes(TENANT_POS_SHIFT);
+ public static final String TRANSACTIONAL = "TRANSACTIONAL";
+ public static final byte[] TRANSACTIONAL_BYTES = Bytes.toBytes(TRANSACTIONAL);
+
private final PhoenixConnection connection;
private final ResultSet emptyResultSet;
public static final int MAX_LOCAL_SI_VERSION_DISALLOW = VersionUtil.encodeVersion("0", "98", "8");
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
index 1a785a6..bb0b1e2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
@@ -162,7 +162,7 @@ public final class PhoenixDriver extends PhoenixEmbeddedDriver {
ConnectionQueryServices connectionQueryServices = connectionQueryServicesMap.get(normalizedConnInfo);
if (connectionQueryServices == null) {
if (normalizedConnInfo.isConnectionless()) {
- connectionQueryServices = new ConnectionlessQueryServicesImpl(services, normalizedConnInfo);
+ connectionQueryServices = new ConnectionlessQueryServicesImpl(services, normalizedConnInfo, info);
} else {
connectionQueryServices = new ConnectionQueryServicesImpl(services, normalizedConnInfo, info);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java
index 3cfaacc..8ee9dbd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java
@@ -187,6 +187,10 @@ public abstract class PhoenixEmbeddedDriver implements Driver, org.apache.phoeni
.setMessage(url).build().buildException();
}
+ public String getZookeeperConnectionString() {
+ return getZookeeperQuorum() + ":" + getPort();
+ }
+
/**
* Detect url with quorum:1,quorum:2 as HBase does not handle different port numbers
* for different quorum hostnames.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index f274336..bacea92 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -34,6 +34,7 @@ import java.text.Format;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -263,6 +264,12 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho
final long startTime = System.currentTimeMillis();
try {
QueryPlan plan = stmt.compilePlan(PhoenixStatement.this, Sequence.ValueOp.VALIDATE_SEQUENCE);
+ // Send mutations to hbase, so they are visible to subsequent reads.
+ // Use original plan for data table so that data and immutable indexes will be sent
+ // TODO: for joins, we need to iterate through all tables, but we need the original table,
+ // not the projected table, so plan.getContext().getResolver().getTables() won't work.
+ Iterator<TableRef> tableRefs = plan.getSourceRefs().iterator();
+ connection.getMutationState().sendUncommitted(tableRefs);
plan = connection.getQueryServices().getOptimizer().optimize(PhoenixStatement.this, plan);
// this will create its own trace internally, so we don't wrap this
// whole thing in tracing
@@ -322,6 +329,12 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho
try {
MutationState state = connection.getMutationState();
MutationPlan plan = stmt.compilePlan(PhoenixStatement.this, Sequence.ValueOp.VALIDATE_SEQUENCE);
+ if (plan.getTargetRef() != null && plan.getTargetRef().getTable() != null && plan.getTargetRef().getTable().isTransactional()) {
+ state.startTransaction();
+ }
+ Iterator<TableRef> tableRefs = plan.getSourceRefs().iterator();
+ state.sendUncommitted(tableRefs);
+ state.checkpointIfNeccessary(plan);
MutationState lastState = plan.execute();
state.join(lastState);
if (connection.getAutoCommit()) {
@@ -1608,4 +1621,4 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
index f13b28e..e8b3389 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
@@ -39,6 +39,7 @@ import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.ServerUtil;
import org.apache.phoenix.util.TrustedByteArrayOutputStream;
import org.apache.phoenix.util.TupleUtil;
@@ -80,7 +81,7 @@ public class HashCacheClient {
*/
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
serialize(ptr, iterator, estimatedSize, onExpressions, singleValueOnly, keyRangeRhsExpression, keyRangeRhsValues);
- return serverCache.addServerCache(keyRanges, ptr, new HashCacheFactory(), cacheUsingTableRef);
+ return serverCache.addServerCache(keyRanges, ptr, ByteUtil.EMPTY_BYTE_ARRAY, new HashCacheFactory(), cacheUsingTableRef);
}
private void serialize(ImmutableBytesWritable ptr, ResultIterator iterator, long estimatedSize, List<Expression> onExpressions, boolean singleValueOnly, Expression keyRangeRhsExpression, List<Expression> keyRangeRhsValues) throws SQLException {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
index 8cae51a..3072736 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
@@ -56,7 +56,7 @@ public class HashCacheFactory implements ServerCacheFactory {
}
@Override
- public Closeable newCache(ImmutableBytesWritable cachePtr, MemoryChunk chunk) throws SQLException {
+ public Closeable newCache(ImmutableBytesWritable cachePtr, byte[] txState, MemoryChunk chunk) throws SQLException {
try {
// This reads the uncompressed length from the front of the compressed input
int uncompressedLen = Snappy.getUncompressedLength(cachePtr.get(), cachePtr.getOffset());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java
index b500a25..d2f91b3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java
@@ -108,7 +108,7 @@ public class PhoenixRecordReader<T extends DBWritable> extends RecordReader<Null
ReadMetricQueue readMetrics = ctx.getReadMetricsQueue();
String tableName = queryPlan.getTableRef().getTable().getPhysicalName().getString();
for (Scan scan : scans) {
- final TableResultIterator tableResultIterator = new TableResultIterator(queryPlan.getContext(),
+ final TableResultIterator tableResultIterator = new TableResultIterator(queryPlan.getContext().getConnection().getMutationState(),
queryPlan.getTableRef(), scan, readMetrics.allotMetric(SCAN_BYTES, tableName));
PeekingResultIterator peekingResultIterator = LookAheadResultIterator.wrap(tableResultIterator);
iterators.add(peekingResultIterator);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectMapper.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectMapper.java
index 32b66f1..3bc3808 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectMapper.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectMapper.java
@@ -138,7 +138,7 @@ public class PhoenixIndexImportDirectMapper extends
private void writeBatch(MutationState mutationState, Context context) throws IOException,
SQLException, InterruptedException {
- final Iterator<Pair<byte[], List<Mutation>>> iterator = mutationState.toMutations(true);
+ final Iterator<Pair<byte[], List<Mutation>>> iterator = mutationState.toMutations(true, null);
while (iterator.hasNext()) {
Pair<byte[], List<Mutation>> mutationPair = iterator.next();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
index bc318d2..c5ff3c6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
@@ -95,17 +95,11 @@ public class QueryOptimizer {
return plans.get(0);
}
- public List<QueryPlan> getBestPlan(PhoenixStatement statement, SelectStatement select, ColumnResolver resolver, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory) throws SQLException {
- return getApplicablePlans(statement, select, resolver, targetColumns, parallelIteratorFactory, true);
+ public List<QueryPlan> getBestPlan(QueryPlan dataPlan, PhoenixStatement statement, SelectStatement select, ColumnResolver resolver, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory) throws SQLException {
+ return getApplicablePlans(dataPlan, statement, targetColumns, parallelIteratorFactory, true);
}
- public List<QueryPlan> getApplicablePlans(PhoenixStatement statement, SelectStatement select, ColumnResolver resolver, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory) throws SQLException {
- return getApplicablePlans(statement, select, resolver, targetColumns, parallelIteratorFactory, false);
- }
-
- private List<QueryPlan> getApplicablePlans(PhoenixStatement statement, SelectStatement select, ColumnResolver resolver, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory, boolean stopAtBestPlan) throws SQLException {
- QueryCompiler compiler = new QueryCompiler(statement, select, resolver, targetColumns, parallelIteratorFactory, new SequenceManager(statement));
- QueryPlan dataPlan = compiler.compile();
+ public List<QueryPlan> getApplicablePlans(QueryPlan dataPlan, PhoenixStatement statement, SelectStatement select, ColumnResolver resolver, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory) throws SQLException {
return getApplicablePlans(dataPlan, statement, targetColumns, parallelIteratorFactory, false);
}
@@ -221,7 +215,8 @@ public class QueryOptimizer {
private static QueryPlan addPlan(PhoenixStatement statement, SelectStatement select, PTable index, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory, QueryPlan dataPlan, boolean isHinted) throws SQLException {
int nColumns = dataPlan.getProjector().getColumnCount();
- String alias = '"' + dataPlan.getTableRef().getTableAlias() + '"'; // double quote in case it's case sensitive
+ String tableAlias = dataPlan.getTableRef().getTableAlias();
+ String alias = tableAlias==null ? null : '"' + tableAlias + '"'; // double quote in case it's case sensitive
String schemaName = index.getParentSchemaName().getString();
schemaName = schemaName.length() == 0 ? null : '"' + schemaName + '"';
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index 3b53309..08c417f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -44,6 +44,8 @@ import org.apache.phoenix.schema.SequenceAllocation;
import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.stats.PTableStats;
+import co.cask.tephra.TransactionSystemClient;
+
public interface ConnectionQueryServices extends QueryServices, MetaDataMutated {
public static final int INITIAL_META_DATA_TABLE_CAPACITY = 100;
@@ -88,7 +90,7 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
void clearTableRegionCache(byte[] tableName) throws SQLException;
- boolean hasInvalidIndexConfiguration();
+ boolean isMutableIndexWALCodecInstalled();
long createSequence(String tenantId, String schemaName, String sequenceName, long startWith, long incrementBy, long cacheSize, long minValue, long maxValue, boolean cycle, long timestamp) throws SQLException;
long dropSequence(String tenantId, String schemaName, String sequenceName, long timestamp) throws SQLException;
@@ -116,4 +118,6 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
public void clearCache() throws SQLException;
public int getSequenceSaltBuckets();
+
+ TransactionSystemClient getTransactionSystemClient();
}
\ No newline at end of file
[27/52] [abbrv] phoenix git commit: PHOENIX-2465 Upgrade ANTLR
version to correct generation (Gabor Liptak)
Posted by ma...@apache.org.
PHOENIX-2465 Upgrade ANTLR version to correct generation (Gabor Liptak)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/21e7c7dd
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/21e7c7dd
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/21e7c7dd
Branch: refs/heads/calcite
Commit: 21e7c7dd20f19e3f2d30387d472940983bf50d02
Parents: bf7187c
Author: James Taylor <jt...@salesforce.com>
Authored: Mon Nov 30 14:10:34 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Nov 30 14:10:34 2015 -0800
----------------------------------------------------------------------
phoenix-core/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/21e7c7dd/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index 5c73ed4..d003e8a 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -112,7 +112,7 @@
<plugin>
<groupId>org.antlr</groupId>
<artifactId>antlr3-maven-plugin</artifactId>
- <version>3.5</version>
+ <version>3.5.2</version>
<executions>
<execution>
<goals>
[36/52] [abbrv] phoenix git commit: Refactoring tests that start the
txn manager to use BaseOwnClusterIT
Posted by ma...@apache.org.
Refactoring tests that start the txn manager to use BaseOwnClusterIT
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/26a6f881
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/26a6f881
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/26a6f881
Branch: refs/heads/calcite
Commit: 26a6f8812d2deb34b47f32a219c9f7b788bdd8d2
Parents: 30ef0cc
Author: Thomas D'Silva <td...@salesforce.com>
Authored: Wed Dec 2 17:40:12 2015 -0800
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Wed Dec 2 17:41:37 2015 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/index/MutableIndexFailureIT.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/26a6f881/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index 9032ee9..979fd31 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -61,6 +61,7 @@ import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.StringUtil;
import org.apache.phoenix.util.TestUtil;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
@@ -117,11 +118,13 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT {
return Arrays.asList(new Boolean[][] { { false }, { true } });
}
+ @Ignore("See PHOENIX-2331")
@Test(timeout=300000)
public void testWriteFailureDisablesLocalIndex() throws Exception {
helpTestWriteFailureDisablesIndex(true);
}
+ @Ignore("See PHOENIX-2332")
@Test(timeout=300000)
public void testWriteFailureDisablesIndex() throws Exception {
helpTestWriteFailureDisablesIndex(false);
[10/52] [abbrv] phoenix git commit: PHOENIX-2450 Cleanup API for
determining if non transactional mutable secondary index configured properly
Posted by ma...@apache.org.
PHOENIX-2450 Cleanup API for determining if non transactional mutable secondary index configured properly
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c98de94f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c98de94f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c98de94f
Branch: refs/heads/calcite
Commit: c98de94f8004566d838de53fb7bc6c150c08e837
Parents: cc9929b
Author: James Taylor <jt...@salesforce.com>
Authored: Tue Nov 24 11:32:17 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Tue Nov 24 11:32:17 2015 -0800
----------------------------------------------------------------------
.../coprocessor/MetaDataEndpointImpl.java | 2 +-
.../phoenix/query/ConnectionQueryServices.java | 2 +-
.../query/ConnectionQueryServicesImpl.java | 26 ++++++++++----------
.../query/ConnectionlessQueryServicesImpl.java | 2 +-
.../query/DelegateConnectionQueryServices.java | 4 +--
.../apache/phoenix/schema/MetaDataClient.java | 8 +++---
.../org/apache/phoenix/util/MetaDataUtil.java | 4 +--
7 files changed, 24 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c98de94f/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 0a313be..7c80f23 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -2674,7 +2674,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
// The last byte is used to communicate whether or not mutable secondary indexing
// was configured properly.
version =
- MetaDataUtil.encodeMutableIndexConfiguredProperly(version,
+ MetaDataUtil.encodeHasIndexWALCodec(version,
IndexManagementUtil.isWALEditCodecSet(this.env.getConfiguration()));
builder.setVersion(version);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c98de94f/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index 08c417f..fc41706 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -90,7 +90,7 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
void clearTableRegionCache(byte[] tableName) throws SQLException;
- boolean isMutableIndexWALCodecInstalled();
+ boolean hasIndexWALCodec();
long createSequence(String tenantId, String schemaName, String sequenceName, long startWith, long incrementBy, long cacheSize, long minValue, long maxValue, boolean cycle, long timestamp) throws SQLException;
long dropSequence(String tenantId, String schemaName, String sequenceName, long timestamp) throws SQLException;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c98de94f/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 2a08f45..df46013 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -173,12 +173,6 @@ import org.apache.twill.zookeeper.ZKClients;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.distributed.PooledClientProvider;
-import co.cask.tephra.distributed.TransactionServiceClient;
-import co.cask.tephra.hbase11.coprocessor.TransactionProcessor;
-
import com.google.common.base.Joiner;
import com.google.common.base.Throwables;
import com.google.common.cache.Cache;
@@ -188,6 +182,12 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import co.cask.tephra.TransactionSystemClient;
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.distributed.PooledClientProvider;
+import co.cask.tephra.distributed.TransactionServiceClient;
+import co.cask.tephra.hbase11.coprocessor.TransactionProcessor;
+
public class ConnectionQueryServicesImpl extends DelegateQueryServices implements ConnectionQueryServices {
private static final Logger logger = LoggerFactory.getLogger(ConnectionQueryServicesImpl.class);
@@ -211,7 +211,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
// Lowest HBase version on the cluster.
private int lowestClusterHBaseVersion = Integer.MAX_VALUE;
- private boolean isMutableIndexWALCodecInstalled = true;
+ private boolean hasIndexWALCodec = true;
@GuardedBy("connectionCountLock")
private int connectionCount = 0;
@@ -1047,11 +1047,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
- private static boolean isInvalidMutableIndexConfig(Long serverVersion) {
+ private static boolean hasIndexWALCodec(Long serverVersion) {
if (serverVersion == null) {
- return false;
+ return true;
}
- return !MetaDataUtil.decodeMutableIndexConfiguredProperly(serverVersion);
+ return MetaDataUtil.decodeHasIndexWALCodec(serverVersion);
}
private static boolean isCompatible(Long serverVersion) {
@@ -1103,7 +1103,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
buf.append(name);
buf.append(';');
}
- isMutableIndexWALCodecInstalled &= !isInvalidMutableIndexConfig(result.getValue());
+ hasIndexWALCodec &= hasIndexWALCodec(result.getValue());
if (minHBaseVersion > MetaDataUtil.decodeHBaseVersion(result.getValue())) {
minHBaseVersion = MetaDataUtil.decodeHBaseVersion(result.getValue());
}
@@ -2485,8 +2485,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
@Override
- public boolean isMutableIndexWALCodecInstalled() {
- return isMutableIndexWALCodecInstalled;
+ public boolean hasIndexWALCodec() {
+ return hasIndexWALCodec;
}
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c98de94f/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 77e4ba9..e983a4c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -390,7 +390,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
}
@Override
- public boolean isMutableIndexWALCodecInstalled() {
+ public boolean hasIndexWALCodec() {
return true;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c98de94f/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index 4952355..ca662be 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -172,8 +172,8 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
}
@Override
- public boolean isMutableIndexWALCodecInstalled() {
- return getDelegate().isMutableIndexWALCodecInstalled();
+ public boolean hasIndexWALCodec() {
+ return getDelegate().hasIndexWALCodec();
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c98de94f/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index cb9b831..29e4af2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -200,8 +200,6 @@ import org.apache.phoenix.util.UpgradeUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import co.cask.tephra.TxConstants;
-
import com.google.common.base.Objects;
import com.google.common.collect.Iterators;
import com.google.common.collect.ListMultimap;
@@ -210,6 +208,8 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
+import co.cask.tephra.TxConstants;
+
public class MetaDataClient {
private static final Logger logger = LoggerFactory.getLogger(MetaDataClient.class);
@@ -1181,7 +1181,7 @@ public class MetaDataClient {
if (hbaseVersion < PhoenixDatabaseMetaData.MUTABLE_SI_VERSION_THRESHOLD) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_MUTABLE_INDEXES).setTableName(indexTableName.getTableName()).build().buildException();
}
- if (!connection.getQueryServices().isMutableIndexWALCodecInstalled() && !dataTable.isTransactional()) {
+ if (!connection.getQueryServices().hasIndexWALCodec() && !dataTable.isTransactional()) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_MUTABLE_INDEX_CONFIG).setTableName(indexTableName.getTableName()).build().buildException();
}
}
@@ -2716,7 +2716,7 @@ public class MetaDataClient {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_MUTABLE_INDEXES)
.setSchemaName(schemaName).setTableName(tableName).build().buildException();
}
- if (connection.getQueryServices().isMutableIndexWALCodecInstalled() && !table.isTransactional()) {
+ if (!connection.getQueryServices().hasIndexWALCodec() && !table.isTransactional()) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_MUTABLE_INDEX_CONFIG)
.setSchemaName(schemaName).setTableName(tableName).build().buildException();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c98de94f/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index 47d3bc3..e50513d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -107,14 +107,14 @@ public class MetaDataUtil {
}
// TODO: generalize this to use two bytes to return a SQL error code instead
- public static long encodeMutableIndexConfiguredProperly(long version, boolean isValid) {
+ public static long encodeHasIndexWALCodec(long version, boolean isValid) {
if (!isValid) {
return version | 1;
}
return version;
}
- public static boolean decodeMutableIndexConfiguredProperly(long version) {
+ public static boolean decodeHasIndexWALCodec(long version) {
return (version & 0xF) == 0;
}
[05/52] [abbrv] phoenix git commit: PHOENIX-1674 Snapshot isolation
transaction support through Tephra (James Taylor, Thomas D'Silva)
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
index d4ef1cf..3444054 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -29,26 +29,33 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import javax.annotation.Nonnull;
import javax.annotation.concurrent.Immutable;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.htrace.Span;
import org.apache.htrace.TraceScope;
import org.apache.phoenix.cache.ServerCacheClient;
import org.apache.phoenix.cache.ServerCacheClient.ServerCache;
+import org.apache.phoenix.compile.MutationPlan;
+import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.index.IndexMaintainer;
import org.apache.phoenix.index.IndexMetaDataCacheClient;
import org.apache.phoenix.index.PhoenixIndexCodec;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixStatement.Operation;
import org.apache.phoenix.monitoring.GlobalClientMetrics;
import org.apache.phoenix.monitoring.MutationMetricQueue;
import org.apache.phoenix.monitoring.MutationMetricQueue.MutationMetric;
@@ -58,9 +65,9 @@ import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.IllegalDataException;
import org.apache.phoenix.schema.MetaDataClient;
import org.apache.phoenix.schema.PColumn;
-import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PRow;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.RowKeySchema;
import org.apache.phoenix.schema.TableRef;
@@ -72,14 +79,25 @@ import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.LogUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.SQLCloseable;
-import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.ServerUtil;
+import org.apache.phoenix.util.TransactionUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import co.cask.tephra.Transaction;
+import co.cask.tephra.Transaction.VisibilityLevel;
+import co.cask.tephra.TransactionAware;
+import co.cask.tephra.TransactionCodec;
+import co.cask.tephra.TransactionContext;
+import co.cask.tephra.TransactionFailureException;
+import co.cask.tephra.TransactionSystemClient;
+import co.cask.tephra.hbase11.TransactionAwareHTable;
+
+import com.google.common.base.Predicate;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
/**
*
@@ -90,44 +108,217 @@ import com.google.common.collect.Maps;
*/
public class MutationState implements SQLCloseable {
private static final Logger logger = LoggerFactory.getLogger(MutationState.class);
-
+ private static final TransactionCodec CODEC = new TransactionCodec();
+
private PhoenixConnection connection;
private final long maxSize;
- private final ImmutableBytesPtr tempPtr = new ImmutableBytesPtr();
private final Map<TableRef, Map<ImmutableBytesPtr,RowMutationState>> mutations;
+ private final List<TransactionAware> txAwares;
+ private final TransactionContext txContext;
+ private final Set<String> uncommittedPhysicalNames = Sets.newHashSetWithExpectedSize(10);
+
+ private Transaction tx;
private long sizeOffset;
private int numRows = 0;
+ private boolean txStarted = false;
+
private final MutationMetricQueue mutationMetricQueue;
private ReadMetricQueue readMetricQueue;
- MutationState(long maxSize, PhoenixConnection connection,
- Map<TableRef, Map<ImmutableBytesPtr, RowMutationState>> mutations) {
- this.maxSize = maxSize;
- this.connection = connection;
- this.mutations = mutations;
- boolean isMetricsEnabled = connection.isRequestLevelMetricsEnabled();
- this.mutationMetricQueue = isMetricsEnabled ? new MutationMetricQueue()
- : NoOpMutationMetricsQueue.NO_OP_MUTATION_METRICS_QUEUE;
- }
-
public MutationState(long maxSize, PhoenixConnection connection) {
- this(maxSize,connection,0);
+ this(maxSize,connection, null);
+ }
+
+ public MutationState(MutationState mutationState) {
+ this(mutationState.maxSize, mutationState.connection, mutationState.getTransaction());
}
public MutationState(long maxSize, PhoenixConnection connection, long sizeOffset) {
- this(maxSize, connection, Maps.<TableRef, Map<ImmutableBytesPtr,RowMutationState>>newHashMapWithExpectedSize(connection.getMutateBatchSize()));
+ this(maxSize, connection, null, sizeOffset);
+ }
+
+ private MutationState(long maxSize, PhoenixConnection connection, Transaction tx) {
+ this(maxSize,connection, tx, 0);
+ }
+
+ private MutationState(long maxSize, PhoenixConnection connection, Transaction tx, long sizeOffset) {
+ this(maxSize, connection, Maps.<TableRef, Map<ImmutableBytesPtr,RowMutationState>>newHashMapWithExpectedSize(connection.getMutateBatchSize()), tx);
this.sizeOffset = sizeOffset;
}
+ MutationState(long maxSize, PhoenixConnection connection,
+ Map<TableRef, Map<ImmutableBytesPtr, RowMutationState>> mutations,
+ Transaction tx) {
+ this.maxSize = maxSize;
+ this.connection = connection;
+ this.mutations = mutations;
+ boolean isMetricsEnabled = connection.isRequestLevelMetricsEnabled();
+ this.mutationMetricQueue = isMetricsEnabled ? new MutationMetricQueue()
+ : NoOpMutationMetricsQueue.NO_OP_MUTATION_METRICS_QUEUE;
+ this.tx = tx;
+ if (tx == null) {
+ this.txAwares = Collections.emptyList();
+ TransactionSystemClient txServiceClient = this.connection
+ .getQueryServices().getTransactionSystemClient();
+ this.txContext = new TransactionContext(txServiceClient);
+ } else {
+ txAwares = Lists.newArrayList();
+ txContext = null;
+ }
+ }
+
public MutationState(TableRef table, Map<ImmutableBytesPtr,RowMutationState> mutations, long sizeOffset, long maxSize, PhoenixConnection connection) {
- this(maxSize, connection, sizeOffset);
+ this(maxSize, connection, null, sizeOffset);
this.mutations.put(table, mutations);
this.numRows = mutations.size();
+ this.tx = connection.getMutationState().getTransaction();
throwIfTooBig();
}
+ public boolean checkpointIfNeccessary(MutationPlan plan) throws SQLException {
+ Transaction currentTx = getTransaction();
+ if (getTransaction() == null || plan.getTargetRef() == null || plan.getTargetRef().getTable() == null || !plan.getTargetRef().getTable().isTransactional()) {
+ return false;
+ }
+ Set<TableRef> sources = plan.getSourceRefs();
+ if (sources.isEmpty()) {
+ return false;
+ }
+ // For a DELETE statement, we're always querying the table being deleted from. This isn't
+ // a problem, but it potentially could be if there are other references to the same table
+ // nested in the DELETE statement (as a sub query or join, for example).
+ TableRef ignoreForExcludeCurrent = plan.getOperation() == Operation.DELETE && sources.size() == 1 ? plan.getTargetRef() : null;
+ boolean excludeCurrent = false;
+ String targetPhysicalName = plan.getTargetRef().getTable().getPhysicalName().getString();
+ for (TableRef source : sources) {
+ if (source.getTable().isTransactional() && !source.equals(ignoreForExcludeCurrent)) {
+ String sourcePhysicalName = source.getTable().getPhysicalName().getString();
+ if (targetPhysicalName.equals(sourcePhysicalName)) {
+ excludeCurrent = true;
+ break;
+ }
+ }
+ }
+ // If we're querying the same table we're updating, we must exclude our writes to
+ // it from being visible.
+ if (excludeCurrent) {
+ // If any source tables have uncommitted data prior to last checkpoint,
+ // then we must create a new checkpoint.
+ boolean hasUncommittedData = false;
+ for (TableRef source : sources) {
+ String sourcePhysicalName = source.getTable().getPhysicalName().getString();
+ if (source.getTable().isTransactional() && uncommittedPhysicalNames.contains(sourcePhysicalName)) {
+ hasUncommittedData = true;
+ break;
+ }
+ }
+ if (hasUncommittedData) {
+ try {
+ if (txContext == null) {
+ currentTx = tx = connection.getQueryServices().getTransactionSystemClient().checkpoint(currentTx);
+ } else {
+ txContext.checkpoint();
+ currentTx = tx = txContext.getCurrentTransaction();
+ }
+ // Since we've checkpointed, we can clear out uncommitted set, since a statement run afterwards
+ // should see all this data.
+ uncommittedPhysicalNames.clear();
+ } catch (TransactionFailureException e) {
+ throw new SQLException(e);
+ }
+ }
+ // Since we're querying our own table while mutating it, we must exclude
+ // see our current mutations, otherwise we can get erroneous results (for DELETE)
+ // or get into an infinite loop (for UPSERT SELECT).
+ currentTx.setVisibility(VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
+ return true;
+ }
+ return false;
+ }
+
+ private void addTransactionParticipant(TransactionAware txAware) throws SQLException {
+ if (txContext == null) {
+ txAwares.add(txAware);
+ assert(tx != null);
+ txAware.startTx(tx);
+ } else {
+ txContext.addTransactionAware(txAware);
+ }
+ }
+
+ // Though MutationState is not thread safe in general, this method should be because it may
+ // be called by TableResultIterator in a multi-threaded manner. Since we do not want to expose
+ // the Transaction outside of MutationState, this seems reasonable, as the member variables
+ // would not change as these threads are running.
+ public HTableInterface getHTable(PTable table) throws SQLException {
+ HTableInterface htable = this.getConnection().getQueryServices().getTable(table.getPhysicalName().getBytes());
+ Transaction currentTx;
+ if (table.isTransactional() && (currentTx=getTransaction()) != null) {
+ TransactionAwareHTable txAware = TransactionUtil.getTransactionAwareHTable(htable, table);
+ // Using cloned mutationState as we may have started a new transaction already
+ // if auto commit is true and we need to use the original one here.
+ txAware.startTx(currentTx);
+ htable = txAware;
+ }
+ return htable;
+ }
+
+ public PhoenixConnection getConnection() {
+ return connection;
+ }
+
+ // Kept private as the Transaction may change when check pointed. Keeping it private ensures
+ // no one holds on to a stale copy.
+ private Transaction getTransaction() {
+ return tx != null ? tx : txContext != null ? txContext.getCurrentTransaction() : null;
+ }
+
+ public boolean isTransactionStarted() {
+ return getTransaction() != null;
+ }
+
+ public long getReadPointer() {
+ Transaction tx = getTransaction();
+ return tx == null ? HConstants.LATEST_TIMESTAMP : tx.getReadPointer();
+ }
+
+ // For testing
+ public long getWritePointer() {
+ Transaction tx = getTransaction();
+ return tx == null ? HConstants.LATEST_TIMESTAMP : tx.getWritePointer();
+ }
+
+ // For testing
+ public VisibilityLevel getVisibilityLevel() {
+ Transaction tx = getTransaction();
+ return tx == null ? null : tx.getVisibilityLevel();
+ }
+
+ public boolean startTransaction() throws SQLException {
+ if (txContext == null) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.NULL_TRANSACTION_CONTEXT).build().buildException();
+ }
+
+ if (connection.getSCN() != null) {
+ throw new SQLExceptionInfo.Builder(
+ SQLExceptionCode.CANNOT_START_TRANSACTION_WITH_SCN_SET)
+ .build().buildException();
+ }
+
+ try {
+ if (!txStarted) {
+ txContext.start();
+ txStarted = true;
+ return true;
+ }
+ } catch (TransactionFailureException e) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.TRANSACTION_FAILED).setRootCause(e).build().buildException();
+ }
+ return false;
+ }
+
public static MutationState emptyMutationState(long maxSize, PhoenixConnection connection) {
- MutationState state = new MutationState(maxSize, connection, Collections.<TableRef, Map<ImmutableBytesPtr,RowMutationState>>emptyMap());
+ MutationState state = new MutationState(maxSize, connection, Collections.<TableRef, Map<ImmutableBytesPtr,RowMutationState>>emptyMap(), null);
state.sizeOffset = 0;
return state;
}
@@ -153,6 +344,13 @@ public class MutationState implements SQLCloseable {
if (this == newMutationState) { // Doesn't make sense
return;
}
+ if (txContext != null) {
+ for (TransactionAware txAware : newMutationState.txAwares) {
+ txContext.addTransactionAware(txAware);
+ }
+ } else {
+ txAwares.addAll(newMutationState.txAwares);
+ }
this.sizeOffset += newMutationState.sizeOffset;
// Merge newMutation with this one, keeping state from newMutation for any overlaps
for (Map.Entry<TableRef, Map<ImmutableBytesPtr,RowMutationState>> entry : newMutationState.mutations.entrySet()) {
@@ -202,7 +400,8 @@ public class MutationState implements SQLCloseable {
throwIfTooBig();
}
- private static ImmutableBytesPtr getNewRowKeyWithRowTimestamp(ImmutableBytesPtr ptr, long rowTimestamp, PTable table) {
+
+ private static ImmutableBytesPtr getNewRowKeyWithRowTimestamp(ImmutableBytesPtr ptr, long rowTimestamp, PTable table) {
RowKeySchema schema = table.getRowKeySchema();
int rowTimestampColPos = table.getRowTimestampColPos();
Field rowTimestampField = schema.getField(rowTimestampColPos);
@@ -223,51 +422,15 @@ public class MutationState implements SQLCloseable {
return ptr;
}
- private Iterator<Pair<byte[],List<Mutation>>> addRowMutations(final TableRef tableRef, final Map<ImmutableBytesPtr, RowMutationState> values, long timestamp, boolean includeMutableIndexes) {
+ private Iterator<Pair<byte[],List<Mutation>>> addRowMutations(final TableRef tableRef, final Map<ImmutableBytesPtr, RowMutationState> values, final long timestamp, boolean includeMutableIndexes, final boolean sendAll) {
final PTable table = tableRef.getTable();
- boolean tableWithRowTimestampCol = table.getRowTimestampColPos() != -1;
final Iterator<PTable> indexes = // Only maintain tables with immutable rows through this client-side mechanism
(table.isImmutableRows() || includeMutableIndexes) ?
IndexMaintainer.nonDisabledIndexIterator(table.getIndexes().iterator()) :
Iterators.<PTable>emptyIterator();
- final List<Mutation> mutations = Lists.newArrayListWithExpectedSize(values.size());
+ final List<Mutation> mutationList = Lists.newArrayListWithExpectedSize(values.size());
final List<Mutation> mutationsPertainingToIndex = indexes.hasNext() ? Lists.<Mutation>newArrayListWithExpectedSize(values.size()) : null;
- Iterator<Map.Entry<ImmutableBytesPtr,RowMutationState>> iterator = values.entrySet().iterator();
- long timestampToUse = timestamp;
- while (iterator.hasNext()) {
- Map.Entry<ImmutableBytesPtr,RowMutationState> rowEntry = iterator.next();
- ImmutableBytesPtr key = rowEntry.getKey();
- RowMutationState state = rowEntry.getValue();
- if (tableWithRowTimestampCol) {
- RowTimestampColInfo rowTsColInfo = state.getRowTimestampColInfo();
- if (rowTsColInfo.useServerTimestamp()) {
- // regenerate the key with this timestamp.
- key = getNewRowKeyWithRowTimestamp(key, timestampToUse, table);
- } else {
- if (rowTsColInfo.getTimestamp() != null) {
- timestampToUse = rowTsColInfo.getTimestamp();
- }
- }
- }
- PRow row = table.newRow(connection.getKeyValueBuilder(), timestampToUse, key);
- List<Mutation> rowMutations, rowMutationsPertainingToIndex;
- if (rowEntry.getValue().getColumnValues() == PRow.DELETE_MARKER) { // means delete
- row.delete();
- rowMutations = row.toRowMutations();
- // Row deletes for index tables are processed by running a re-written query
- // against the index table (as this allows for flexibility in being able to
- // delete rows).
- rowMutationsPertainingToIndex = Collections.emptyList();
- } else {
- for (Map.Entry<PColumn,byte[]> valueEntry : rowEntry.getValue().getColumnValues().entrySet()) {
- row.setValue(valueEntry.getKey(), valueEntry.getValue());
- }
- rowMutations = row.toRowMutations();
- rowMutationsPertainingToIndex = rowMutations;
- }
- mutations.addAll(rowMutations);
- if (mutationsPertainingToIndex != null) mutationsPertainingToIndex.addAll(rowMutationsPertainingToIndex);
- }
+ generateMutations(tableRef, timestamp, values, mutationList, mutationsPertainingToIndex);
return new Iterator<Pair<byte[],List<Mutation>>>() {
boolean isFirst = true;
@@ -280,14 +443,24 @@ public class MutationState implements SQLCloseable {
public Pair<byte[], List<Mutation>> next() {
if (isFirst) {
isFirst = false;
- return new Pair<byte[],List<Mutation>>(table.getPhysicalName().getBytes(),mutations);
+ return new Pair<byte[],List<Mutation>>(table.getPhysicalName().getBytes(), mutationList);
}
PTable index = indexes.next();
List<Mutation> indexMutations;
try {
indexMutations =
IndexUtil.generateIndexData(table, index, mutationsPertainingToIndex,
- tempPtr, connection.getKeyValueBuilder(), connection);
+ connection.getKeyValueBuilder(), connection);
+ // we may also have to include delete mutations for immutable tables if we are not processing all the tables in the mutations map
+ if (!sendAll) {
+ TableRef key = new TableRef(index);
+ Map<ImmutableBytesPtr, RowMutationState> rowToColumnMap = mutations.remove(key);
+ if (rowToColumnMap!=null) {
+ final List<Mutation> deleteMutations = Lists.newArrayList();
+ generateMutations(tableRef, timestamp, rowToColumnMap, deleteMutations, null);
+ indexMutations.addAll(deleteMutations);
+ }
+ }
} catch (SQLException e) {
throw new IllegalDataException(e);
}
@@ -301,28 +474,84 @@ public class MutationState implements SQLCloseable {
};
}
+
+ private void generateMutations(final TableRef tableRef, long timestamp,
+ final Map<ImmutableBytesPtr, RowMutationState> values,
+ final List<Mutation> mutationList, final List<Mutation> mutationsPertainingToIndex) {
+ final PTable table = tableRef.getTable();
+ boolean tableWithRowTimestampCol = table.getRowTimestampColPos() != -1;
+ Iterator<Map.Entry<ImmutableBytesPtr, RowMutationState>> iterator =
+ values.entrySet().iterator();
+ long timestampToUse = timestamp;
+ while (iterator.hasNext()) {
+ Map.Entry<ImmutableBytesPtr, RowMutationState> rowEntry = iterator.next();
+ ImmutableBytesPtr key = rowEntry.getKey();
+ RowMutationState state = rowEntry.getValue();
+ if (tableWithRowTimestampCol) {
+ RowTimestampColInfo rowTsColInfo = state.getRowTimestampColInfo();
+ if (rowTsColInfo.useServerTimestamp()) {
+ // regenerate the key with this timestamp.
+ key = getNewRowKeyWithRowTimestamp(key, timestampToUse, table);
+ } else {
+ if (rowTsColInfo.getTimestamp() != null) {
+ timestampToUse = rowTsColInfo.getTimestamp();
+ }
+ }
+ }
+ PRow row =
+ tableRef.getTable()
+ .newRow(connection.getKeyValueBuilder(), timestampToUse, key);
+ List<Mutation> rowMutations, rowMutationsPertainingToIndex;
+ if (rowEntry.getValue().getColumnValues() == PRow.DELETE_MARKER) { // means delete
+ row.delete();
+ rowMutations = row.toRowMutations();
+ // Row deletes for index tables are processed by running a re-written query
+ // against the index table (as this allows for flexibility in being able to
+ // delete rows).
+ rowMutationsPertainingToIndex = Collections.emptyList();
+ } else {
+ for (Map.Entry<PColumn, byte[]> valueEntry : rowEntry.getValue().getColumnValues()
+ .entrySet()) {
+ row.setValue(valueEntry.getKey(), valueEntry.getValue());
+ }
+ rowMutations = row.toRowMutations();
+ rowMutationsPertainingToIndex = rowMutations;
+ }
+ mutationList.addAll(rowMutations);
+ if (mutationsPertainingToIndex != null) mutationsPertainingToIndex
+ .addAll(rowMutationsPertainingToIndex);
+ }
+ }
/**
* Get the unsorted list of HBase mutations for the tables with uncommitted data.
* @return list of HBase mutations for uncommitted data.
*/
+ public Iterator<Pair<byte[],List<Mutation>>> toMutations(Long timestamp) {
+ return toMutations(false, timestamp);
+ }
+
public Iterator<Pair<byte[],List<Mutation>>> toMutations() {
- return toMutations(false);
+ return toMutations(false, null);
}
public Iterator<Pair<byte[],List<Mutation>>> toMutations(final boolean includeMutableIndexes) {
+ return toMutations(includeMutableIndexes, null);
+ }
+
+ public Iterator<Pair<byte[],List<Mutation>>> toMutations(final boolean includeMutableIndexes, final Long tableTimestamp) {
final Iterator<Map.Entry<TableRef, Map<ImmutableBytesPtr,RowMutationState>>> iterator = this.mutations.entrySet().iterator();
if (!iterator.hasNext()) {
return Iterators.emptyIterator();
}
Long scn = connection.getSCN();
- final long timestamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
+ final long timestamp = (tableTimestamp!=null && tableTimestamp!=QueryConstants.UNSET_TIMESTAMP) ? tableTimestamp : (scn == null ? HConstants.LATEST_TIMESTAMP : scn);
return new Iterator<Pair<byte[],List<Mutation>>>() {
private Map.Entry<TableRef, Map<ImmutableBytesPtr,RowMutationState>> current = iterator.next();
private Iterator<Pair<byte[],List<Mutation>>> innerIterator = init();
private Iterator<Pair<byte[],List<Mutation>>> init() {
- return addRowMutations(current.getKey(), current.getValue(), timestamp, includeMutableIndexes);
+ return addRowMutations(current.getKey(), current.getValue(), timestamp, includeMutableIndexes, true);
}
@Override
@@ -346,58 +575,62 @@ public class MutationState implements SQLCloseable {
};
}
- /**
- * Validates that the meta data is valid against the server meta data if we haven't yet done so.
- * Otherwise, for every UPSERT VALUES call, we'd need to hit the server to see if the meta data
- * has changed.
- * @param connection
- * @return the server time to use for the upsert
- * @throws SQLException if the table or any columns no longer exist
- */
- private long[] validate() throws SQLException {
- int i = 0;
- Long scn = connection.getSCN();
- MetaDataClient client = new MetaDataClient(connection);
- long[] timeStamps = new long[this.mutations.size()];
- for (Map.Entry<TableRef, Map<ImmutableBytesPtr,RowMutationState>> entry : mutations.entrySet()) {
- TableRef tableRef = entry.getKey();
- long serverTimeStamp = tableRef.getTimeStamp();
- PTable table = tableRef.getTable();
- // If we're auto committing, we've already validated the schema when we got the ColumnResolver,
- // so no need to do it again here.
- if (!connection.getAutoCommit()) {
- MetaDataMutationResult result = client.updateCache(table.getSchemaName().getString(), table.getTableName().getString());
- long timestamp = result.getMutationTime();
- if (timestamp != QueryConstants.UNSET_TIMESTAMP) {
- serverTimeStamp = timestamp;
- if (result.wasUpdated()) {
- // TODO: use bitset?
- table = result.getTable();
- PColumn[] columns = new PColumn[table.getColumns().size()];
- for (Map.Entry<ImmutableBytesPtr,RowMutationState> rowEntry : entry.getValue().entrySet()) {
- RowMutationState valueEntry = rowEntry.getValue();
- if (valueEntry != null) {
- Map<PColumn, byte[]> colValues = valueEntry.getColumnValues();
- if (colValues != PRow.DELETE_MARKER) {
- for (PColumn column : colValues.keySet()) {
- columns[column.getPosition()] = column;
- }
- }
- }
+ /**
+ * Validates that the meta data is valid against the server meta data if we haven't yet done so.
+ * Otherwise, for every UPSERT VALUES call, we'd need to hit the server to see if the meta data
+ * has changed.
+ * @param connection
+ * @return the server time to use for the upsert
+ * @throws SQLException if the table or any columns no longer exist
+ */
+ private long[] validateAll() throws SQLException {
+ int i = 0;
+ long[] timeStamps = new long[this.mutations.size()];
+ for (Map.Entry<TableRef, Map<ImmutableBytesPtr,RowMutationState>> entry : mutations.entrySet()) {
+ TableRef tableRef = entry.getKey();
+ timeStamps[i++] = validate(tableRef, entry.getValue());
+ }
+ return timeStamps;
+ }
+
+ private long validate(TableRef tableRef, Map<ImmutableBytesPtr, RowMutationState> rowKeyToColumnMap) throws SQLException {
+ Long scn = connection.getSCN();
+ MetaDataClient client = new MetaDataClient(connection);
+ long serverTimeStamp = tableRef.getTimeStamp();
+ PTable table = tableRef.getTable();
+ // If we're auto committing, we've already validated the schema when we got the ColumnResolver,
+ // so no need to do it again here.
+ if (!connection.getAutoCommit()) {
+ MetaDataMutationResult result = client.updateCache(table.getSchemaName().getString(), table.getTableName().getString());
+ long timestamp = result.getMutationTime();
+ if (timestamp != QueryConstants.UNSET_TIMESTAMP) {
+ serverTimeStamp = timestamp;
+ if (result.wasUpdated()) {
+ // TODO: use bitset?
+ table = result.getTable();
+ PColumn[] columns = new PColumn[table.getColumns().size()];
+ for (Map.Entry<ImmutableBytesPtr,RowMutationState> rowEntry : rowKeyToColumnMap.entrySet()) {
+ RowMutationState valueEntry = rowEntry.getValue();
+ if (valueEntry != null) {
+ Map<PColumn, byte[]> colValues = valueEntry.getColumnValues();
+ if (colValues != PRow.DELETE_MARKER) {
+ for (PColumn column : colValues.keySet()) {
+ columns[column.getPosition()] = column;
+ }
+ }
}
- for (PColumn column : columns) {
- if (column != null) {
- table.getColumnFamily(column.getFamilyName().getString()).getColumn(column.getName().getString());
- }
+ }
+ for (PColumn column : columns) {
+ if (column != null) {
+ table.getColumnFamily(column.getFamilyName().getString()).getColumn(column.getName().getString());
}
- tableRef.setTable(table);
}
+ tableRef.setTable(table);
}
}
- timeStamps[i++] = scn == null ? serverTimeStamp == QueryConstants.UNSET_TIMESTAMP ? HConstants.LATEST_TIMESTAMP : serverTimeStamp : scn;
}
- return timeStamps;
- }
+ return scn == null ? serverTimeStamp == QueryConstants.UNSET_TIMESTAMP ? HConstants.LATEST_TIMESTAMP : serverTimeStamp : scn;
+ }
private static long calculateMutationSize(List<Mutation> mutations) {
long byteSize = 0;
@@ -410,149 +643,292 @@ public class MutationState implements SQLCloseable {
return byteSize;
}
+ private boolean hasKeyValueColumn(PTable table, PTable index) {
+ IndexMaintainer maintainer = index.getIndexMaintainer(table, connection);
+ return !maintainer.getAllColumns().isEmpty();
+ }
+
+ private void divideImmutableIndexes(Iterator<PTable> enabledImmutableIndexes, PTable table, List<PTable> rowKeyIndexes, List<PTable> keyValueIndexes) {
+ while (enabledImmutableIndexes.hasNext()) {
+ PTable index = enabledImmutableIndexes.next();
+ if (index.getIndexType() != IndexType.LOCAL) {
+ if (hasKeyValueColumn(table, index)) {
+ keyValueIndexes.add(index);
+ } else {
+ rowKeyIndexes.add(index);
+ }
+ }
+ }
+ }
+ private class MetaDataAwareHTable extends DelegateHTable {
+ private final TableRef tableRef;
+
+ private MetaDataAwareHTable(HTableInterface delegate, TableRef tableRef) {
+ super(delegate);
+ this.tableRef = tableRef;
+ }
+
+ /**
+ * Called by Tephra when a transaction is aborted. We have this wrapper so that we get an
+ * opportunity to attach our index meta data to the mutations such that we can also undo
+ * the index mutations.
+ */
+ @Override
+ public void delete(List<Delete> deletes) throws IOException {
+ try {
+ PTable table = tableRef.getTable();
+ List<PTable> indexes = table.getIndexes();
+ Iterator<PTable> enabledIndexes = IndexMaintainer.nonDisabledIndexIterator(indexes.iterator());
+ if (enabledIndexes.hasNext()) {
+ List<PTable> keyValueIndexes = Collections.emptyList();
+ ImmutableBytesWritable indexMetaDataPtr = new ImmutableBytesWritable();
+ boolean attachMetaData = table.getIndexMaintainers(indexMetaDataPtr, connection);
+ if (table.isImmutableRows()) {
+ List<PTable> rowKeyIndexes = Lists.newArrayListWithExpectedSize(indexes.size());
+ keyValueIndexes = Lists.newArrayListWithExpectedSize(indexes.size());
+ divideImmutableIndexes(enabledIndexes, table, rowKeyIndexes, keyValueIndexes);
+ // Generate index deletes for immutable indexes that only reference row key
+ // columns and submit directly here.
+ ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+ for (PTable index : rowKeyIndexes) {
+ List<Delete> indexDeletes = IndexUtil.generateDeleteIndexData(table, index, deletes, ptr, connection.getKeyValueBuilder(), connection);
+ HTableInterface hindex = connection.getQueryServices().getTable(index.getPhysicalName().getBytes());
+ hindex.delete(indexDeletes);
+ }
+ }
+
+ // If we have mutable indexes, local immutable indexes, or global immutable indexes
+ // that reference key value columns, setup index meta data and attach here. In this
+ // case updates to the indexes will be generated on the server side.
+ // An alternative would be to let Tephra track the row keys for the immutable index
+ // by adding it as a transaction participant (soon we can prevent any conflict
+ // detection from occurring) with the downside being the additional memory required.
+ if (!keyValueIndexes.isEmpty()) {
+ attachMetaData = true;
+ IndexMaintainer.serializeAdditional(table, indexMetaDataPtr, keyValueIndexes, connection);
+ }
+ if (attachMetaData) {
+ setMetaDataOnMutations(tableRef, deletes, indexMetaDataPtr);
+ }
+ }
+ delegate.delete(deletes);
+ } catch (SQLException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
@SuppressWarnings("deprecation")
- public void commit() throws SQLException {
+ private void send(Iterator<TableRef> tableRefIterator) throws SQLException {
int i = 0;
- PName tenantId = connection.getTenantId();
- long[] serverTimeStamps = validate();
- Iterator<Map.Entry<TableRef, Map<ImmutableBytesPtr,RowMutationState>>> iterator = this.mutations.entrySet().iterator();
+ long[] serverTimeStamps = null;
+ boolean sendAll = false;
+ if (tableRefIterator == null) {
+ serverTimeStamps = validateAll();
+ tableRefIterator = mutations.keySet().iterator();
+ sendAll = true;
+ }
+
// add tracing for this operation
try (TraceScope trace = Tracing.startNewSpan(connection, "Committing mutations to tables")) {
Span span = trace.getSpan();
- while (iterator.hasNext()) {
- Map.Entry<TableRef, Map<ImmutableBytesPtr,RowMutationState>> entry = iterator.next();
- // at this point we are going through mutations for each table
-
- Map<ImmutableBytesPtr,RowMutationState> valuesMap = entry.getValue();
- // above is mutations for a table where the first part is the row key and the second part is column values.
-
- TableRef tableRef = entry.getKey();
- PTable table = tableRef.getTable();
- table.getIndexMaintainers(tempPtr, connection);
- boolean hasIndexMaintainers = tempPtr.getLength() > 0;
- boolean isDataTable = true;
- long serverTimestamp = serverTimeStamps[i++];
- Iterator<Pair<byte[],List<Mutation>>> mutationsIterator = addRowMutations(tableRef, valuesMap, serverTimestamp, false);
- // above returns an iterator of pair where the first
- while (mutationsIterator.hasNext()) {
- Pair<byte[],List<Mutation>> pair = mutationsIterator.next();
- byte[] htableName = pair.getFirst();
- List<Mutation> mutations = pair.getSecond();
-
- //create a span per target table
- //TODO maybe we can be smarter about the table name to string here?
- Span child = Tracing.child(span,"Writing mutation batch for table: "+Bytes.toString(htableName));
-
- int retryCount = 0;
- boolean shouldRetry = false;
- do {
- ServerCache cache = null;
- if (hasIndexMaintainers && isDataTable) {
- byte[] attribValue = null;
- byte[] uuidValue;
- if (IndexMetaDataCacheClient.useIndexMetadataCache(connection, mutations, tempPtr.getLength())) {
- IndexMetaDataCacheClient client = new IndexMetaDataCacheClient(connection, tableRef);
- cache = client.addIndexMetadataCache(mutations, tempPtr);
- child.addTimelineAnnotation("Updated index metadata cache");
- uuidValue = cache.getId();
- // If we haven't retried yet, retry for this case only, as it's possible that
- // a split will occur after we send the index metadata cache to all known
- // region servers.
- shouldRetry = true;
- } else {
- attribValue = ByteUtil.copyKeyBytesIfNecessary(tempPtr);
- uuidValue = ServerCacheClient.generateId();
- }
- // Either set the UUID to be able to access the index metadata from the cache
- // or set the index metadata directly on the Mutation
- for (Mutation mutation : mutations) {
- if (tenantId != null) {
- byte[] tenantIdBytes = ScanUtil.getTenantIdBytes(
- table.getRowKeySchema(),
- table.getBucketNum()!=null,
- tenantId);
- mutation.setAttribute(PhoenixRuntime.TENANT_ID_ATTRIB, tenantIdBytes);
- }
- mutation.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
- if (attribValue != null) {
- mutation.setAttribute(PhoenixIndexCodec.INDEX_MD, attribValue);
- }
- }
- }
-
- SQLException sqlE = null;
- HTableInterface hTable = connection.getQueryServices().getTable(htableName);
- try {
- long numMutations = mutations.size();
+ ImmutableBytesWritable indexMetaDataPtr = new ImmutableBytesWritable();
+ while (tableRefIterator.hasNext()) {
+ // at this point we are going through mutations for each table
+ TableRef tableRef = tableRefIterator.next();
+ Map<ImmutableBytesPtr, RowMutationState> valuesMap = mutations.get(tableRef);
+ if (valuesMap == null || valuesMap.isEmpty()) {
+ continue;
+ }
+ PTable table = tableRef.getTable();
+ // Track tables to which we've sent uncommitted data
+ if (table.isTransactional()) {
+ uncommittedPhysicalNames.add(table.getPhysicalName().getString());
+ }
+ table.getIndexMaintainers(indexMetaDataPtr, connection);
+ boolean isDataTable = true;
+ // Validate as we go if transactional since we can undo if a problem occurs (which is unlikely)
+ long serverTimestamp = serverTimeStamps == null ? validate(tableRef, valuesMap) : serverTimeStamps[i++];
+ Iterator<Pair<byte[],List<Mutation>>> mutationsIterator = addRowMutations(tableRef, valuesMap, serverTimestamp, false, sendAll);
+ while (mutationsIterator.hasNext()) {
+ Pair<byte[],List<Mutation>> pair = mutationsIterator.next();
+ byte[] htableName = pair.getFirst();
+ List<Mutation> mutationList = pair.getSecond();
+
+ //create a span per target table
+ //TODO maybe we can be smarter about the table name to string here?
+ Span child = Tracing.child(span,"Writing mutation batch for table: "+Bytes.toString(htableName));
+
+ int retryCount = 0;
+ boolean shouldRetry = false;
+ do {
+ ServerCache cache = null;
+ if (isDataTable) {
+ cache = setMetaDataOnMutations(tableRef, mutationList, indexMetaDataPtr);
+ }
+
+ // If we haven't retried yet, retry for this case only, as it's possible that
+ // a split will occur after we send the index metadata cache to all known
+ // region servers.
+ shouldRetry = cache != null;
+ SQLException sqlE = null;
+ HTableInterface hTable = connection.getQueryServices().getTable(htableName);
+ try {
+ if (table.isTransactional()) {
+ // If we have indexes, wrap the HTable in a delegate HTable that
+ // will attach the necessary index meta data in the event of a
+ // rollback
+ if (!table.getIndexes().isEmpty()) {
+ hTable = new MetaDataAwareHTable(hTable, tableRef);
+ }
+ TransactionAwareHTable txnAware = TransactionUtil.getTransactionAwareHTable(hTable, table);
+ // Don't add immutable indexes (those are the only ones that would participate
+ // during a commit), as we don't need conflict detection for these.
+ if (isDataTable) {
+ // Even for immutable, we need to do this so that an abort has the state
+ // necessary to generate the rows to delete.
+ addTransactionParticipant(txnAware);
+ } else {
+ txnAware.startTx(getTransaction());
+ }
+ hTable = txnAware;
+ }
+ long numMutations = mutationList.size();
GLOBAL_MUTATION_BATCH_SIZE.update(numMutations);
long startTime = System.currentTimeMillis();
- child.addTimelineAnnotation("Attempt " + retryCount);
- hTable.batch(mutations);
- child.stop();
+ child.addTimelineAnnotation("Attempt " + retryCount);;
+ hTable.batch(mutationList);
+ child.stop();
+ child.stop();
shouldRetry = false;
long mutationCommitTime = System.currentTimeMillis() - startTime;
GLOBAL_MUTATION_COMMIT_TIME.update(mutationCommitTime);
- long mutationSizeBytes = calculateMutationSize(mutations);
+ long mutationSizeBytes = calculateMutationSize(mutationList);
MutationMetric mutationsMetric = new MutationMetric(numMutations, mutationSizeBytes, mutationCommitTime);
mutationMetricQueue.addMetricsForTable(Bytes.toString(htableName), mutationsMetric);
- } catch (Exception e) {
- SQLException inferredE = ServerUtil.parseServerExceptionOrNull(e);
- if (inferredE != null) {
- if (shouldRetry && retryCount == 0 && inferredE.getErrorCode() == SQLExceptionCode.INDEX_METADATA_NOT_FOUND.getErrorCode()) {
- // Swallow this exception once, as it's possible that we split after sending the index metadata
- // and one of the region servers doesn't have it. This will cause it to have it the next go around.
- // If it fails again, we don't retry.
- String msg = "Swallowing exception and retrying after clearing meta cache on connection. " + inferredE;
- logger.warn(LogUtil.addCustomAnnotations(msg, connection));
- connection.getQueryServices().clearTableRegionCache(htableName);
+ } catch (Exception e) {
+ SQLException inferredE = ServerUtil.parseServerExceptionOrNull(e);
+ if (inferredE != null) {
+ if (shouldRetry && retryCount == 0 && inferredE.getErrorCode() == SQLExceptionCode.INDEX_METADATA_NOT_FOUND.getErrorCode()) {
+ // Swallow this exception once, as it's possible that we split after sending the index metadata
+ // and one of the region servers doesn't have it. This will cause it to have it the next go around.
+ // If it fails again, we don't retry.
+ String msg = "Swallowing exception and retrying after clearing meta cache on connection. " + inferredE;
+ logger.warn(LogUtil.addCustomAnnotations(msg, connection));
+ connection.getQueryServices().clearTableRegionCache(htableName);
+
+ // add a new child span as this one failed
+ child.addTimelineAnnotation(msg);
+ child.stop();
+ child = Tracing.child(span,"Failed batch, attempting retry");
+
+ continue;
+ }
+ e = inferredE;
+ }
+ // Throw to client with both what was committed so far and what is left to be committed.
+ // That way, client can either undo what was done or try again with what was not done.
+ sqlE = new CommitException(e, getUncommittedStatementIndexes());
+ } finally {
+ try {
+ if (cache != null) {
+ cache.close();
+ }
+ } finally {
+ try {
+ hTable.close();
+ }
+ catch (IOException e) {
+ if (sqlE != null) {
+ sqlE.setNextException(ServerUtil.parseServerException(e));
+ } else {
+ sqlE = ServerUtil.parseServerException(e);
+ }
+ }
+ if (sqlE != null) {
+ // clear pending mutations
+ mutations.clear();
+ throw sqlE;
+ }
+ }
+ }
+ } while (shouldRetry && retryCount++ < 1);
+ isDataTable = false;
+ }
+ if (tableRef.getTable().getType() != PTableType.INDEX) {
+ numRows -= valuesMap.size();
+ }
+ // Remove batches as we process them
+ if (sendAll) {
+ tableRefIterator.remove(); // Iterating through actual map in this case
+ } else {
+ mutations.remove(tableRef);
+ }
+ }
+ }
+ // Note that we cannot assume that *all* mutations have been sent, since we've optimized this
+ // now to only send the mutations for the tables we're querying, hence we've removed the
+ // assertions that we're here before.
+ }
- // add a new child span as this one failed
- child.addTimelineAnnotation(msg);
- child.stop();
- child = Tracing.child(span,"Failed batch, attempting retry");
+ public byte[] encodeTransaction() throws SQLException {
+ try {
+ return CODEC.encode(getTransaction());
+ } catch (IOException e) {
+ throw new SQLException(e);
+ }
+ }
+
+ public static Transaction decodeTransaction(byte[] txnBytes) throws IOException {
+ return (txnBytes == null || txnBytes.length==0) ? null : CODEC.decode(txnBytes);
+ }
- continue;
- }
- e = inferredE;
- }
- sqlE = new CommitException(e, getUncommittedStatementIndexes());
- } finally {
- try {
- hTable.close();
- } catch (IOException e) {
- if (sqlE != null) {
- sqlE.setNextException(ServerUtil.parseServerException(e));
- } else {
- sqlE = ServerUtil.parseServerException(e);
- }
- } finally {
- try {
- if (cache != null) {
- cache.close();
- }
- } finally {
- if (sqlE != null) {
- throw sqlE;
- }
- }
- }
- }
- } while (shouldRetry && retryCount++ < 1);
- isDataTable = false;
- }
- if (tableRef.getTable().getType() != PTableType.INDEX) {
- numRows -= entry.getValue().size();
+ private ServerCache setMetaDataOnMutations(TableRef tableRef, List<? extends Mutation> mutations,
+ ImmutableBytesWritable indexMetaDataPtr) throws SQLException {
+ PTable table = tableRef.getTable();
+ byte[] tenantId = connection.getTenantId() == null ? null : connection.getTenantId().getBytes();
+ ServerCache cache = null;
+ byte[] attribValue = null;
+ byte[] uuidValue = null;
+ byte[] txState = ByteUtil.EMPTY_BYTE_ARRAY;
+ if (table.isTransactional()) {
+ txState = encodeTransaction();
+ }
+ boolean hasIndexMetaData = indexMetaDataPtr.getLength() > 0;
+ if (hasIndexMetaData) {
+ if (IndexMetaDataCacheClient.useIndexMetadataCache(connection, mutations, indexMetaDataPtr.getLength() + txState.length)) {
+ IndexMetaDataCacheClient client = new IndexMetaDataCacheClient(connection, tableRef);
+ cache = client.addIndexMetadataCache(mutations, indexMetaDataPtr, txState);
+ uuidValue = cache.getId();
+ } else {
+ attribValue = ByteUtil.copyKeyBytesIfNecessary(indexMetaDataPtr);
+ uuidValue = ServerCacheClient.generateId();
+ }
+ } else if (txState.length == 0) {
+ return null;
+ }
+ // Either set the UUID to be able to access the index metadata from the cache
+ // or set the index metadata directly on the Mutation
+ for (Mutation mutation : mutations) {
+ if (tenantId != null) {
+ mutation.setAttribute(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
+ }
+ mutation.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
+ if (attribValue != null) {
+ mutation.setAttribute(PhoenixIndexCodec.INDEX_MD, attribValue);
+ if (txState.length > 0) {
+ mutation.setAttribute(BaseScannerRegionObserver.TX_STATE, txState);
}
- iterator.remove(); // Remove batches as we process them
+ } else if (!hasIndexMetaData && txState.length > 0) {
+ mutation.setAttribute(BaseScannerRegionObserver.TX_STATE, txState);
}
}
- assert(numRows==0);
- assert(this.mutations.isEmpty());
+ return cache;
}
- public void rollback(PhoenixConnection connection) throws SQLException {
+ private void clear() throws SQLException {
this.mutations.clear();
numRows = 0;
}
@@ -570,6 +946,100 @@ public class MutationState implements SQLCloseable {
@Override
public void close() throws SQLException {
}
+
+ private void reset() {
+ txStarted = false;
+ tx = null;
+ uncommittedPhysicalNames.clear();
+ }
+
+ public void rollback() throws SQLException {
+ clear();
+ txAwares.clear();
+ if (txContext != null) {
+ try {
+ if (txStarted) {
+ txContext.abort();
+ }
+ } catch (TransactionFailureException e) {
+ throw new SQLException(e); // TODO: error code
+ } finally {
+ reset();
+ }
+ }
+ }
+
+ public void commit() throws SQLException {
+ boolean sendMutationsFailed=false;
+ try {
+ send();
+ } catch (Throwable t) {
+ sendMutationsFailed=true;
+ throw t;
+ } finally {
+ txAwares.clear();
+ if (txContext != null) {
+ try {
+ if (txStarted && !sendMutationsFailed) {
+ txContext.finish();
+ }
+ } catch (TransactionFailureException e) {
+ try {
+ txContext.abort(e);
+ // abort and throw the original commit failure exception
+ throw TransactionUtil.getTransactionFailureException(e);
+ } catch (TransactionFailureException e1) {
+ // if abort fails and throw the abort failure exception
+ throw TransactionUtil.getTransactionFailureException(e1);
+ }
+ } finally {
+ if (!sendMutationsFailed) {
+ reset();
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Support read-your-own-write semantics by sending uncommitted data to HBase prior to running a
+ * query. In this way, they are visible to subsequent reads but are not actually committed until
+ * commit is called.
+ * @param tableRefs
+ * @return true if at least partially transactional and false otherwise.
+ * @throws SQLException
+ */
+ public boolean sendUncommitted(Iterator<TableRef> tableRefs) throws SQLException {
+ Transaction currentTx = getTransaction();
+ if (currentTx != null) {
+ // Initialize visibility so that transactions see their own writes.
+ // The checkpoint() method will set it to not see writes if necessary.
+ currentTx.setVisibility(VisibilityLevel.SNAPSHOT);
+ }
+ Iterator<TableRef> filteredTableRefs = Iterators.filter(tableRefs, new Predicate<TableRef>(){
+ @Override
+ public boolean apply(TableRef tableRef) {
+ return tableRef.getTable().isTransactional();
+ }
+ });
+ if (filteredTableRefs.hasNext()) {
+ // FIXME: strip table alias to prevent equality check from failing due to alias mismatch on null alias.
+ // We really should be keying the tables based on the physical table name.
+ List<TableRef> strippedAliases = Lists.newArrayListWithExpectedSize(mutations.keySet().size());
+ while (filteredTableRefs.hasNext()) {
+ TableRef tableRef = filteredTableRefs.next();
+ strippedAliases.add(new TableRef(null, tableRef.getTable(), tableRef.getTimeStamp(), tableRef.getLowerBoundTimeStamp(), tableRef.hasDynamicCols()));
+ }
+ startTransaction();
+ send(strippedAliases.iterator());
+ return true;
+ }
+ return false;
+ }
+
+ public void send() throws SQLException {
+ send(null);
+ }
public static int[] joinSortedIntArrays(int[] a, int[] b) {
int[] result = new int[a.length + b.length];
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
index 3098980..19b3e6b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
@@ -151,7 +151,7 @@ public class ScanPlan extends BaseQueryPlan {
return spoolingResultIteratorFactory;
} else {
return new ChunkedResultIterator.ChunkedResultIteratorFactory(
- spoolingResultIteratorFactory, table);
+ spoolingResultIteratorFactory, context.getConnection().getMutationState(), table);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
index dea82a8..ab7534a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
@@ -89,6 +89,8 @@ import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TrustedByteArrayOutputStream;
+import co.cask.tephra.TxConstants;
+
import com.google.common.base.Predicate;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
@@ -874,7 +876,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> {
}
else if (kv.getTypeByte() == KeyValue.Type.DeleteFamily.getCode()
// Since we don't include the index rows in the change set for txn tables, we need to detect row deletes that have transformed by TransactionProcessor
- ) {
+ || (CellUtil.matchingQualifier(kv, TxConstants.FAMILY_DELETE_QUALIFIER) && CellUtil.matchingValue(kv, HConstants.EMPTY_BYTE_ARRAY))) {
nDeleteCF++;
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java
index 70ddc86..05a01b9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java
@@ -24,7 +24,6 @@ import java.util.List;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
import org.apache.phoenix.cache.ServerCacheClient;
import org.apache.phoenix.cache.ServerCacheClient.ServerCache;
import org.apache.phoenix.compile.ScanRanges;
@@ -59,7 +58,7 @@ public class IndexMetaDataCacheClient {
* @param mutations the list of mutations that will be sent in a batch to server
* @param indexMetaDataByteLength length in bytes of the index metadata cache
*/
- public static boolean useIndexMetadataCache(PhoenixConnection connection, List<Mutation> mutations, int indexMetaDataByteLength) {
+ public static boolean useIndexMetadataCache(PhoenixConnection connection, List<? extends Mutation> mutations, int indexMetaDataByteLength) {
ReadOnlyProps props = connection.getQueryServices().getProps();
int threshold = props.getInt(INDEX_MUTATE_BATCH_SIZE_THRESHOLD_ATTRIB, QueryServicesOptions.DEFAULT_INDEX_MUTATE_BATCH_SIZE_THRESHOLD);
return (indexMetaDataByteLength > ServerCacheClient.UUID_LENGTH && mutations.size() > threshold);
@@ -72,25 +71,26 @@ public class IndexMetaDataCacheClient {
* @throws MaxServerCacheSizeExceededException if size of hash cache exceeds max allowed
* size
*/
- public ServerCache addIndexMetadataCache(List<Mutation> mutations, ImmutableBytesWritable ptr) throws SQLException {
+ public ServerCache addIndexMetadataCache(List<? extends Mutation> mutations, ImmutableBytesWritable ptr, byte[] txState) throws SQLException {
/**
* Serialize and compress hashCacheTable
*/
- return serverCache.addServerCache(ScanUtil.newScanRanges(mutations), ptr, new IndexMetaDataCacheFactory(), cacheUsingTableRef);
+ return serverCache.addServerCache(ScanUtil.newScanRanges(mutations), ptr, txState, new IndexMetaDataCacheFactory(), cacheUsingTableRef);
}
/**
* Send the index metadata cahce to all region servers for regions that will handle the mutations.
+ * @param txState TODO
* @return client-side {@link ServerCache} representing the added index metadata cache
* @throws SQLException
* @throws MaxServerCacheSizeExceededException if size of hash cache exceeds max allowed
* size
*/
- public ServerCache addIndexMetadataCache(ScanRanges ranges, ImmutableBytesWritable ptr) throws SQLException {
+ public ServerCache addIndexMetadataCache(ScanRanges ranges, ImmutableBytesWritable ptr, byte[] txState) throws SQLException {
/**
* Serialize and compress hashCacheTable
*/
- return serverCache.addServerCache(ranges, ptr, new IndexMetaDataCacheFactory(), cacheUsingTableRef);
+ return serverCache.addServerCache(ranges, ptr, txState, new IndexMetaDataCacheFactory(), cacheUsingTableRef);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheFactory.java
index 488db44..7ced2f8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheFactory.java
@@ -24,11 +24,15 @@ import java.io.IOException;
import java.sql.SQLException;
import java.util.List;
+import co.cask.tephra.Transaction;
+
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.cache.IndexMetaDataCache;
import org.apache.phoenix.coprocessor.ServerCachingProtocol.ServerCacheFactory;
+import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
import org.apache.phoenix.memory.MemoryManager.MemoryChunk;
+import org.apache.phoenix.util.TransactionUtil;
public class IndexMetaDataCacheFactory implements ServerCacheFactory {
public IndexMetaDataCacheFactory() {
@@ -43,10 +47,16 @@ public class IndexMetaDataCacheFactory implements ServerCacheFactory {
}
@Override
- public Closeable newCache (ImmutableBytesWritable cachePtr, final MemoryChunk chunk) throws SQLException {
+ public Closeable newCache (ImmutableBytesWritable cachePtr, byte[] txState, final MemoryChunk chunk) throws SQLException {
// just use the standard keyvalue builder - this doesn't really need to be fast
- final List<IndexMaintainer> maintainers =
+ final List<IndexMaintainer> maintainers =
IndexMaintainer.deserialize(cachePtr, GenericKeyValueBuilder.INSTANCE);
+ final Transaction txn;
+ try {
+ txn = txState.length!=0 ? MutationState.decodeTransaction(txState) : null;
+ } catch (IOException e) {
+ throw new SQLException(e);
+ }
return new IndexMetaDataCache() {
@Override
@@ -58,6 +68,11 @@ public class IndexMetaDataCacheFactory implements ServerCacheFactory {
public List<IndexMaintainer> getIndexMaintainers() {
return maintainers;
}
+
+ @Override
+ public Transaction getTransaction() {
+ return txn;
+ }
};
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
index 2fd168a..806a20a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
@@ -75,7 +75,7 @@ import com.google.common.collect.Multimap;
*
* @since 2.1
*/
-public class PhoenixIndexFailurePolicy extends KillServerOnFailurePolicy {
+public class PhoenixIndexFailurePolicy extends KillServerOnFailurePolicy {
private static final Log LOG = LogFactory.getLog(PhoenixIndexFailurePolicy.class);
private RegionCoprocessorEnvironment env;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaData.java
index 3dcc44e..60ae915 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexMetaData.java
@@ -22,6 +22,8 @@ import java.sql.SQLException;
import java.util.List;
import java.util.Map;
+import co.cask.tephra.Transaction;
+
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.cache.GlobalCache;
@@ -46,8 +48,10 @@ public class PhoenixIndexMetaData implements IndexMetaData {
byte[] uuid = attributes.get(PhoenixIndexCodec.INDEX_UUID);
if (uuid == null) { return IndexMetaDataCache.EMPTY_INDEX_META_DATA_CACHE; }
byte[] md = attributes.get(PhoenixIndexCodec.INDEX_MD);
+ byte[] txState = attributes.get(BaseScannerRegionObserver.TX_STATE);
if (md != null) {
final List<IndexMaintainer> indexMaintainers = IndexMaintainer.deserialize(md);
+ final Transaction txn = MutationState.decodeTransaction(txState);
return new IndexMetaDataCache() {
@Override
@@ -58,6 +62,11 @@ public class PhoenixIndexMetaData implements IndexMetaData {
return indexMaintainers;
}
+ @Override
+ public Transaction getTransaction() {
+ return txn;
+ }
+
};
} else {
byte[] tenantIdBytes = attributes.get(PhoenixRuntime.TENANT_ID_ATTRIB);
@@ -80,6 +89,10 @@ public class PhoenixIndexMetaData implements IndexMetaData {
this.attributes = attributes;
}
+ public Transaction getTransaction() {
+ return indexMetaDataCache.getTransaction();
+ }
+
public List<IndexMaintainer> getIndexMaintainers() {
return indexMetaDataCache.getIndexMaintainers();
}
[08/52] [abbrv] phoenix git commit: PHOENIX-1674 Snapshot isolation
transaction support through Tephra (James Taylor, Thomas D'Silva)
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
index b269212..8f5301c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
@@ -52,17 +52,28 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
protected final boolean localIndex;
private final String tableDDLOptions;
+ private final String tableName;
+ private final String indexName;
+ private final String fullTableName;
+ private final String fullIndexName;
- public MutableIndexIT(boolean localIndex) {
+ public MutableIndexIT(boolean localIndex, boolean transactional) {
this.localIndex = localIndex;
StringBuilder optionBuilder = new StringBuilder();
+ if (transactional) {
+ optionBuilder.append("TRANSACTIONAL=true");
+ }
this.tableDDLOptions = optionBuilder.toString();
+ this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + ( transactional ? "_TXN" : "");
+ this.indexName = "IDX" + ( transactional ? "_TXN" : "");
+ this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ this.fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
}
- @Parameters(name="localIndex = {0}")
+ @Parameters(name="localIndex = {0} , transactional = {1}")
public static Collection<Boolean[]> data() {
return Arrays.asList(new Boolean[][] {
- { false }, {true}
+ { false, false }, { false, true }, { true, false }, { true, true }
});
}
@@ -71,12 +82,6 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
createMultiCFTestTable(fullTableName, tableDDLOptions);
populateMultiCFTestTable(fullTableName);
PreparedStatement stmt = conn.prepareStatement("CREATE " + (localIndex ? " LOCAL " : "") + " INDEX " + indexName + " ON " + fullTableName
@@ -174,11 +179,6 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
conn.createStatement().execute("CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)" + tableDDLOptions);
query = "SELECT * FROM " + fullTableName;
rs = conn.createStatement().executeQuery(query);
@@ -284,12 +284,6 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
// make sure that the tables are empty, but reachable
conn.createStatement().execute("CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)" + tableDDLOptions);
query = "SELECT * FROM " + fullTableName;
@@ -405,12 +399,6 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
// make sure that the tables are empty, but reachable
conn.createStatement().execute(
"CREATE TABLE " + fullTableName
@@ -487,24 +475,20 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
@Test
public void testUpsertingNullForIndexedColumns() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String testTableName = tableName + "_" + System.currentTimeMillis();
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexeName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
Statement stmt = conn.createStatement();
- stmt.execute("CREATE TABLE " + fullTableName + "(v1 VARCHAR PRIMARY KEY, v2 DOUBLE, v3 VARCHAR) "+tableDDLOptions);
- stmt.execute("CREATE " + (localIndex ? "LOCAL" : "") + " INDEX " + indexName + " ON " + fullTableName + " (v2) INCLUDE(v3)");
+ stmt.execute("CREATE TABLE " + testTableName + "(v1 VARCHAR PRIMARY KEY, v2 DOUBLE, v3 VARCHAR) "+tableDDLOptions);
+ stmt.execute("CREATE " + (localIndex ? "LOCAL" : "") + " INDEX " + indexName + " ON " + testTableName + " (v2) INCLUDE(v3)");
//create a row with value null for indexed column v2
- stmt.executeUpdate("upsert into " + fullTableName + " values('cc1', null, 'abc')");
+ stmt.executeUpdate("upsert into " + testTableName + " values('cc1', null, 'abc')");
conn.commit();
//assert values in index table
- rs = stmt.executeQuery("select * from " + fullIndexeName);
+ rs = stmt.executeQuery("select * from " + fullIndexName);
assertTrue(rs.next());
assertEquals(0, Doubles.compare(0, rs.getDouble(1)));
assertTrue(rs.wasNull());
@@ -513,7 +497,7 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
//assert values in data table
- rs = stmt.executeQuery("select v1, v2, v3 from " + fullTableName);
+ rs = stmt.executeQuery("select v1, v2, v3 from " + testTableName);
assertTrue(rs.next());
assertEquals("cc1", rs.getString(1));
assertEquals(0, Doubles.compare(0, rs.getDouble(2)));
@@ -522,11 +506,11 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
//update the previously null value for indexed column v2 to a non-null value 1.23
- stmt.executeUpdate("upsert into " + fullTableName + " values('cc1', 1.23, 'abc')");
+ stmt.executeUpdate("upsert into " + testTableName + " values('cc1', 1.23, 'abc')");
conn.commit();
//assert values in data table
- rs = stmt.executeQuery("select /*+ NO_INDEX */ v1, v2, v3 from " + fullTableName);
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ v1, v2, v3 from " + testTableName);
assertTrue(rs.next());
assertEquals("cc1", rs.getString(1));
assertEquals(0, Doubles.compare(1.23, rs.getDouble(2)));
@@ -534,7 +518,7 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
//assert values in index table
- rs = stmt.executeQuery("select * from " + fullIndexeName);
+ rs = stmt.executeQuery("select * from " + indexName);
assertTrue(rs.next());
assertEquals(0, Doubles.compare(1.23, rs.getDouble(1)));
assertEquals("cc1", rs.getString(2));
@@ -542,11 +526,11 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
//update the value for indexed column v2 back to null
- stmt.executeUpdate("upsert into " + fullTableName + " values('cc1', null, 'abc')");
+ stmt.executeUpdate("upsert into " + testTableName + " values('cc1', null, 'abc')");
conn.commit();
//assert values in index table
- rs = stmt.executeQuery("select * from " + fullIndexeName);
+ rs = stmt.executeQuery("select * from " + indexName);
assertTrue(rs.next());
assertEquals(0, Doubles.compare(0, rs.getDouble(1)));
assertTrue(rs.wasNull());
@@ -555,7 +539,7 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
//assert values in data table
- rs = stmt.executeQuery("select v1, v2, v3 from " + fullTableName);
+ rs = stmt.executeQuery("select v1, v2, v3 from " + testTableName);
assertTrue(rs.next());
assertEquals("cc1", rs.getString(1));
assertEquals(0, Doubles.compare(0, rs.getDouble(2)));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java
new file mode 100644
index 0000000..58f6226
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java
@@ -0,0 +1,510 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end.index.txn;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.end2end.Shadower;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import com.google.common.collect.Maps;
+
+@RunWith(Parameterized.class)
+public class MutableRollbackIT extends BaseHBaseManagedTimeIT {
+
+ private final boolean localIndex;
+ private String tableName1;
+ private String indexName1;
+ private String fullTableName1;
+ private String tableName2;
+ private String indexName2;
+ private String fullTableName2;
+
+ public MutableRollbackIT(boolean localIndex) {
+ this.localIndex = localIndex;
+ this.tableName1 = TestUtil.DEFAULT_DATA_TABLE_NAME + "_1_";
+ this.indexName1 = "IDX1";
+ this.fullTableName1 = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName1);
+ this.tableName2 = TestUtil.DEFAULT_DATA_TABLE_NAME + "_2_";
+ this.indexName2 = "IDX2";
+ this.fullTableName2 = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName2);
+ }
+
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(2);
+ props.put(QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @Parameters(name="localIndex = {0}")
+ public static Collection<Boolean> data() {
+ return Arrays.asList(new Boolean[] {
+ false, true
+ });
+ }
+
+ public void testRollbackOfUncommittedExistingKeyValueIndexUpdate() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName1 + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
+ stmt.execute("CREATE TABLE " + fullTableName2 + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) IMMUTABLE_ROWS=true");
+ stmt.execute("CREATE "+(localIndex? " LOCAL " : "")+"INDEX " + indexName1 + " ON " + fullTableName1 + " (v1) INCLUDE(v2)");
+ stmt.execute("CREATE "+(localIndex? " LOCAL " : "")+"INDEX " + indexName2 + " ON " + fullTableName2 + " (v1) INCLUDE(v2)");
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'y', 'a')");
+ conn.commit();
+
+ //assert rows exists in fullTableName1
+ ResultSet rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert rows exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert no rows exists in fullTableName2
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ //assert no rows exists in indexName2
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName2 + ")*/ k, v1 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'y', 'b')");
+ stmt.executeUpdate("upsert into " + fullTableName2 + " values('a', 'b', 'c')");
+
+ //assert new covered column value
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("b", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert new covered column value
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("b", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert rows exists in fullTableName2
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName2);
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals("b", rs.getString(2));
+ assertEquals("c", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert rows exists in " + fullTableName2 + " index table
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName2 + ")*/ k, v1 from " + fullTableName2);
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals("b", rs.getString(2));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ //assert original row exists in fullTableName1
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert original row exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert no rows exists in fullTableName2
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ //assert no rows exists in indexName2
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName2 + ")*/ k, v1 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'z', 'a')");
+ stmt.executeUpdate("upsert into " + fullTableName2 + " values('a', 'b', 'c')");
+ conn.commit();
+
+ assertDataAndIndexRows(stmt);
+ stmt.executeUpdate("delete from " + fullTableName1 + " where k='x'");
+ stmt.executeUpdate("delete from " + fullTableName2 + " where v1='b'");
+
+ //assert no rows exists in fullTableName1
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName1);
+ assertFalse(rs.next());
+ //assert no rows exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertFalse(rs.next());
+
+ //assert no rows exists in fullTableName2
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName2);
+ assertFalse(rs.next());
+ //assert no rows exists in indexName2
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName2 + ")*/ k, v1 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ conn.rollback();
+ assertDataAndIndexRows(stmt);
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testRollbackOfUncommittedExistingRowKeyIndexUpdate() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName1 + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
+ stmt.execute("CREATE TABLE " + fullTableName2 + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) IMMUTABLE_ROWS=true");
+ stmt.execute("CREATE "+(localIndex? " LOCAL " : "")+"INDEX " + indexName1 + " ON " + fullTableName1 + " (v1, k)");
+ stmt.execute("CREATE "+(localIndex? " LOCAL " : "")+"INDEX " + indexName2 + " ON " + fullTableName2 + " (v1, k)");
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'y', 'a')");
+ conn.commit();
+
+ //assert rows exists in " + fullTableName1 + "
+ ResultSet rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert rows exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertFalse(rs.next());
+
+ //assert no rows exists in fullTableName2
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ //assert no rows exists in indexName2
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName2 + ")*/ k, v1 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'z', 'a')");
+ stmt.executeUpdate("upsert into " + fullTableName2 + " values('a', 'b', 'c')");
+
+ assertDataAndIndexRows(stmt);
+
+ conn.rollback();
+
+ //assert original row exists in fullTableName1
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert original row exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert no rows exists in fullTableName2
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ //assert no rows exists in indexName2
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'z', 'a')");
+ stmt.executeUpdate("upsert into " + fullTableName2 + " values('a', 'b', 'c')");
+ conn.commit();
+
+ assertDataAndIndexRows(stmt);
+ stmt.executeUpdate("delete from " + fullTableName1 + " where k='x'");
+ stmt.executeUpdate("delete from " + fullTableName2 + " where v1='b'");
+
+ //assert no rows exists in fullTableName1
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertFalse(rs.next());
+ //assert no rows exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1);
+ assertFalse(rs.next());
+
+ //assert no rows exists in fullTableName2
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName2);
+ assertFalse(rs.next());
+ //assert no rows exists in indexName2
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName2 + ")*/ k, v1 from " + fullTableName2);
+ assertFalse(rs.next());
+
+ conn.rollback();
+ assertDataAndIndexRows(stmt);
+
+ } finally {
+ conn.close();
+ }
+ }
+
+ private void assertDataAndIndexRows(Statement stmt) throws SQLException, IOException {
+ ResultSet rs;
+ //assert new covered row key value exists in fullTableName1
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("z", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert new covered row key value exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("z", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert rows exists in fullTableName2
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName2);
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals("b", rs.getString(2));
+ assertEquals("c", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert rows exists in " + fullTableName2 + " index table
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName2);
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals("b", rs.getString(2));
+ assertFalse(rs.next());
+ }
+
+ @Test
+ public void testMultiRollbackOfUncommittedExistingRowKeyIndexUpdate() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName1 + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
+ stmt.execute("CREATE "+(localIndex? " LOCAL " : "")+"INDEX " + indexName1 + " ON " + fullTableName1 + " (v1, k)");
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'yyyy', 'a')");
+ conn.commit();
+
+ //assert rows exists in " + fullTableName1 + "
+ ResultSet rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("yyyy", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert rows exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("yyyy", rs.getString(2));
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'zzz', 'a')");
+
+ //assert new covered row key value exists in fullTableName1
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("zzz", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert new covered row key value exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("zzz", rs.getString(2));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ //assert original row exists in fullTableName1
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("yyyy", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert original row exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("yyyy", rs.getString(2));
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'zz', 'a')");
+
+ //assert new covered row key value exists in fullTableName1
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("zz", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert new covered row key value exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("zz", rs.getString(2));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ //assert original row exists in fullTableName1
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("yyyy", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert original row exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("yyyy", rs.getString(2));
+ assertFalse(rs.next());
+
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testCheckpointAndRollback() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName1 + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
+ stmt.execute("CREATE "+(localIndex? " LOCAL " : "")+"INDEX " + indexName1 + " ON " + fullTableName1 + " (v1)");
+ stmt.executeUpdate("upsert into " + fullTableName1 + " values('x', 'a', 'a')");
+ conn.commit();
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + "(k,v1) SELECT k,v1||'a' FROM " + fullTableName1);
+ ResultSet rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("aa", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("aa", rs.getString(2));
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName1 + "(k,v1) SELECT k,v1||'a' FROM " + fullTableName1);
+
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("aaa", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("aaa", rs.getString(2));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ //assert original row exists in fullTableName1
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName1);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("a", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert original row exists in indexName1
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName1 + ")*/ k, v1 from " + fullTableName1 + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("a", rs.getString(2));
+ assertFalse(rs.next());
+
+ } finally {
+ conn.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java
new file mode 100644
index 0000000..dbddcb1
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end.index.txn;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.end2end.Shadower;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import com.google.common.collect.Maps;
+
+@RunWith(Parameterized.class)
+public class RollbackIT extends BaseHBaseManagedTimeIT {
+
+ private final boolean localIndex;
+ private final boolean mutable;
+ private final String tableName;
+ private final String indexName;
+ private final String fullTableName;
+
+ public RollbackIT(boolean localIndex, boolean mutable) {
+ this.localIndex = localIndex;
+ this.mutable = mutable;
+ this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME;
+ this.indexName = "IDX";
+ this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ }
+
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(2);
+ props.put(QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @Parameters(name="localIndex = {0} , mutable = {1}")
+ public static Collection<Boolean[]> data() {
+ return Arrays.asList(new Boolean[][] {
+ { false, false }, { false, true },
+ { true, false }, { true, true }
+ });
+ }
+
+ @Test
+ public void testRollbackOfUncommittedKeyValueIndexInsert() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"+(!mutable? " IMMUTABLE_ROWS=true" : ""));
+ stmt.execute("CREATE "+(localIndex? "LOCAL " : "")+"INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE(v2)");
+
+ stmt.executeUpdate("upsert into " + fullTableName + " values('x', 'y', 'a')");
+
+ //assert values in data table
+ ResultSet rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert values in index table
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName + ")*/ k, v1, v2 from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ //assert values in data table
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName);
+ assertFalse(rs.next());
+
+ //assert values in index table
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName + ")*/ k, v1, v2 from " + fullTableName);
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testRollbackOfUncommittedRowKeyIndexInsert() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName + "(k VARCHAR, v1 VARCHAR, v2 VARCHAR, CONSTRAINT pk PRIMARY KEY (v1, v2))"+(!mutable? " IMMUTABLE_ROWS=true" : ""));
+ stmt.execute("CREATE "+(localIndex? "LOCAL " : "")+"INDEX " + indexName + " ON " + fullTableName + "(v1, k)");
+
+ stmt.executeUpdate("upsert into " + fullTableName + " values('x', 'y', 'a')");
+
+ ResultSet rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName);
+
+ //assert values in data table
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert values in index table
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName + ")*/ k, v1 from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("y", rs.getString(2));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ //assert values in data table
+ rs = stmt.executeQuery("select /*+ NO_INDEX */ k, v1, v2 from " + fullTableName);
+ assertFalse(rs.next());
+
+ //assert values in index table
+ rs = stmt.executeQuery("select /*+ INDEX(" + indexName + ")*/ k, v1 from " + fullTableName);
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java
new file mode 100644
index 0000000..0e16d97
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end.index.txn;
+
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR;
+import static org.apache.phoenix.util.PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PARAM;
+import static org.apache.phoenix.util.TestUtil.LOCALHOST;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
+import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.hbase.index.Indexer;
+import org.apache.phoenix.jdbc.PhoenixTestDriver;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import com.google.common.collect.Maps;
+
+@RunWith(Parameterized.class)
+public class TxWriteFailureIT extends BaseTest {
+
+ private static PhoenixTestDriver driver;
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ private static final String SCHEMA_NAME = "S";
+ private static final String DATA_TABLE_NAME = "T";
+ private static final String INDEX_TABLE_NAME = "I";
+ private static final String DATA_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, DATA_TABLE_NAME);
+ private static final String INDEX_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, INDEX_TABLE_NAME);
+ private static final String ROW_TO_FAIL = "fail";
+
+ private final boolean localIndex;
+ private final boolean mutable;
+
+ public TxWriteFailureIT(boolean localIndex, boolean mutable) {
+ this.localIndex = localIndex;
+ this.mutable = mutable;
+ }
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+ setUpConfigForMiniCluster(conf);
+ conf.setClass("hbase.coprocessor.region.classes", FailingRegionObserver.class, RegionObserver.class);
+ conf.setBoolean("hbase.coprocessor.abortonerror", false);
+ conf.setBoolean(Indexer.CHECK_VERSION_CONF_KEY, false);
+ TEST_UTIL.startMiniCluster();
+ String clientPort = TEST_UTIL.getConfiguration().get(
+ QueryServices.ZOOKEEPER_PORT_ATTRIB);
+ url = JDBC_PROTOCOL + JDBC_PROTOCOL_SEPARATOR + LOCALHOST
+ + JDBC_PROTOCOL_SEPARATOR + clientPort
+ + JDBC_PROTOCOL_TERMINATOR + PHOENIX_TEST_DRIVER_URL_PARAM;
+
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+ // Must update config before starting server
+ props.put(QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ driver = initAndRegisterDriver(url, new ReadOnlyProps(props.entrySet().iterator()));
+ clusterInitialized = true;
+ setupTxManager();
+ }
+
+ @Parameters(name="localIndex = {0} , mutable = {1}")
+ public static Collection<Boolean[]> data() {
+ return Arrays.asList(new Boolean[][] {
+ { false, false }, { false, true }, { true, false }, { true, true }
+ });
+ }
+
+ @Test
+ public void testIndexTableWriteFailure() throws Exception {
+ helpTestWriteFailure(true);
+ }
+
+ @Test
+ public void testDataTableWriteFailure() throws Exception {
+ helpTestWriteFailure(false);
+ }
+
+ private void helpTestWriteFailure(boolean indexTableWriteFailure) throws SQLException {
+ ResultSet rs;
+
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = driver.connect(url, props);
+ conn.setAutoCommit(false);
+ conn.createStatement().execute(
+ "CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR PRIMARY KEY, v1 VARCHAR)"+(!mutable? " IMMUTABLE_ROWS=true" : ""));
+ conn.createStatement().execute(
+ "CREATE "+(localIndex? "LOCAL " : "")+"INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v1)");
+
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?)");
+ // to create a data table write failure set k as the ROW_TO_FAIL, to create an index table write failure set v1 as the ROW_TO_FAIL,
+ // FailingRegionObserver will throw an exception if the put contains ROW_TO_FAIL
+ stmt.setString(1, !indexTableWriteFailure ? ROW_TO_FAIL : "k1");
+ stmt.setString(2, indexTableWriteFailure ? ROW_TO_FAIL : "k2");
+ stmt.execute();
+ stmt.setString(1, "k2");
+ stmt.setString(2, "v2");
+ stmt.execute();
+ try {
+ conn.commit();
+ fail();
+ }
+ catch (Exception e) {
+ conn.rollback();
+ }
+ stmt.setString(1, "k3");
+ stmt.setString(2, "v3");
+ stmt.execute();
+ //this should pass
+ conn.commit();
+
+ // verify that only k3,v3 exists in the data table
+ String dataSql = "SELECT k, v1 FROM " + DATA_TABLE_FULL_NAME + " order by k";
+ rs = conn.createStatement().executeQuery("EXPLAIN "+dataSql);
+ assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER S.T",
+ QueryUtil.getExplainPlan(rs));
+ rs = conn.createStatement().executeQuery(dataSql);
+ assertTrue(rs.next());
+ assertEquals("k3", rs.getString(1));
+ assertEquals("v3", rs.getString(2));
+ assertFalse(rs.next());
+
+ // verify the only k3,v3 exists in the index table
+ String indexSql = "SELECT k, v1 FROM " + DATA_TABLE_FULL_NAME + " order by v1";
+ rs = conn.createStatement().executeQuery("EXPLAIN "+indexSql);
+ if(localIndex) {
+ assertEquals(
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _LOCAL_IDX_" + DATA_TABLE_FULL_NAME + " [-32768]\n" +
+ " SERVER FILTER BY FIRST KEY ONLY\n" +
+ "CLIENT MERGE SORT",
+ QueryUtil.getExplainPlan(rs));
+ } else {
+ assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME + "\n SERVER FILTER BY FIRST KEY ONLY",
+ QueryUtil.getExplainPlan(rs));
+ }
+ rs = conn.createStatement().executeQuery(indexSql);
+ assertTrue(rs.next());
+ assertEquals("k3", rs.getString(1));
+ assertEquals("v3", rs.getString(2));
+ assertFalse(rs.next());
+
+ conn.createStatement().execute("DROP TABLE " + DATA_TABLE_FULL_NAME);
+ }
+
+
+ public static class FailingRegionObserver extends SimpleRegionObserver {
+ @Override
+ public void prePut(ObserverContext<RegionCoprocessorEnvironment> c, Put put, WALEdit edit,
+ final Durability durability) throws HBaseIOException {
+ if (shouldFailUpsert(c, put)) {
+ // throwing anything other than instances of IOException result
+ // in this coprocessor being unloaded
+ // DoNotRetryIOException tells HBase not to retry this mutation
+ // multiple times
+ throw new DoNotRetryIOException();
+ }
+ }
+
+ private boolean shouldFailUpsert(ObserverContext<RegionCoprocessorEnvironment> c, Put put) {
+ return Bytes.contains(put.getRow(), Bytes.toBytes(ROW_TO_FAIL));
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java b/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
index e0f0a3c..4b06834 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
@@ -22,8 +22,6 @@ package org.apache.phoenix.execute;
import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Sets.newHashSet;
import static java.util.Collections.singletonList;
-import static org.apache.phoenix.query.BaseTest.initAndRegisterDriver;
-import static org.apache.phoenix.query.BaseTest.setUpConfigForMiniCluster;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR;
@@ -39,6 +37,8 @@ import java.sql.Driver;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
@@ -57,38 +58,37 @@ import org.apache.hadoop.hbase.coprocessor.RegionObserver;
import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.hbase.index.Indexer;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.TableRef;
-import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
-@Category(NeedsOwnMiniClusterTest.class)
-public class PartialCommitIT {
+@RunWith(Parameterized.class)
+public class PartialCommitIT extends BaseTest {
- private static final String TABLE_NAME_TO_FAIL = "b_failure_table".toUpperCase();
- private static final byte[] ROW_TO_FAIL = Bytes.toBytes("fail me");
- private static final String UPSERT_TO_FAIL = "upsert into " + TABLE_NAME_TO_FAIL + " values ('" + Bytes.toString(ROW_TO_FAIL) + "', 'boom!')";
- private static final String UPSERT_SELECT_TO_FAIL = "upsert into " + TABLE_NAME_TO_FAIL + " select k, c from a_success_table";
- private static final String DELETE_TO_FAIL = "delete from " + TABLE_NAME_TO_FAIL + " where k='z'";
+ private final String A_SUCESS_TABLE;
+ private final String B_FAILURE_TABLE;
+ private final String C_SUCESS_TABLE;
+ private final String UPSERT_TO_FAIL;
+ private final String UPSERT_SELECT_TO_FAIL;
+ private final String DELETE_TO_FAIL;
+ private static final String TABLE_NAME_TO_FAIL = "B_FAILURE_TABLE";
+ private static final byte[] ROW_TO_FAIL_UPSERT_BYTES = Bytes.toBytes("fail me upsert");
+ private static final byte[] ROW_TO_FAIL_DELETE_BYTES = Bytes.toBytes("fail me delete");
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- private static String url;
private static Driver driver;
- private static final Properties props = new Properties();
-
- static {
- props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, 10);
- }
@BeforeClass
public static void setupCluster() throws Exception {
@@ -104,29 +104,50 @@ public class PartialCommitIT {
Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
// Must update config before starting server
- props.put(QueryServices.DROP_METADATA_ATTRIB, Boolean.toString(true));
- driver = initAndRegisterDriver(url, new ReadOnlyProps(props.entrySet().iterator()));
+ driver = initAndRegisterDriver(url, ReadOnlyProps.EMPTY_PROPS);
+ clusterInitialized = true;
+ setupTxManager();
createTablesWithABitOfData();
}
+ @Parameters(name="transactional = {0}")
+ public static Collection<Boolean> data() {
+ return Arrays.asList(false, true);
+ }
+
+ public PartialCommitIT(boolean transactional) {
+ if (transactional) {
+ A_SUCESS_TABLE = "A_SUCCESS_TABLE_TXN";
+ B_FAILURE_TABLE = TABLE_NAME_TO_FAIL+"_TXN";
+ C_SUCESS_TABLE = "C_SUCCESS_TABLE_TXN";
+ }
+ else {
+ A_SUCESS_TABLE = "A_SUCCESS_TABLE";
+ B_FAILURE_TABLE = TABLE_NAME_TO_FAIL;
+ C_SUCESS_TABLE = "C_SUCCESS_TABLE";
+ }
+ UPSERT_TO_FAIL = "upsert into " + B_FAILURE_TABLE + " values ('" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "', 'boom!')";
+ UPSERT_SELECT_TO_FAIL = "upsert into " + B_FAILURE_TABLE + " select k, c from a_success_table";
+ DELETE_TO_FAIL = "delete from " + B_FAILURE_TABLE + " where k='" + Bytes.toString(ROW_TO_FAIL_DELETE_BYTES) + "'";
+ }
+
private static void createTablesWithABitOfData() throws Exception {
- Properties props = new Properties();
- props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, 10);
-
try (Connection con = driver.connect(url, new Properties())) {
Statement sta = con.createStatement();
sta.execute("create table a_success_table (k varchar primary key, c varchar)");
sta.execute("create table b_failure_table (k varchar primary key, c varchar)");
sta.execute("create table c_success_table (k varchar primary key, c varchar)");
+ sta.execute("create table a_success_table_txn (k varchar primary key, c varchar) TRANSACTIONAL=true");
+ sta.execute("create table b_failure_table_txn (k varchar primary key, c varchar) TRANSACTIONAL=true");
+ sta.execute("create table c_success_table_txn (k varchar primary key, c varchar) TRANSACTIONAL=true");
con.commit();
}
- props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, 100);
-
try (Connection con = driver.connect(url, new Properties())) {
con.setAutoCommit(false);
Statement sta = con.createStatement();
- for (String table : newHashSet("a_success_table", TABLE_NAME_TO_FAIL, "c_success_table")) {
+ for (String table : newHashSet("a_success_table", "b_failure_table", "c_success_table",
+ "a_success_table_txn", "b_failure_table_txn", "c_success_table_txn")) {
sta.execute("upsert into " + table + " values ('z', 'z')");
sta.execute("upsert into " + table + " values ('zz', 'zz')");
sta.execute("upsert into " + table + " values ('zzz', 'zzz')");
@@ -142,46 +163,44 @@ public class PartialCommitIT {
@Test
public void testNoFailure() {
- testPartialCommit(singletonList("upsert into a_success_table values ('testNoFailure', 'a')"), 0, new int[0], false,
- singletonList("select count(*) from a_success_table where k='testNoFailure'"), singletonList(new Integer(1)));
+ testPartialCommit(singletonList("upsert into " + A_SUCESS_TABLE + " values ('testNoFailure', 'a')"), 0, new int[0], false,
+ singletonList("select count(*) from " + A_SUCESS_TABLE + " where k='testNoFailure'"), singletonList(new Integer(1)));
}
@Test
public void testUpsertFailure() {
- testPartialCommit(newArrayList("upsert into a_success_table values ('testUpsertFailure1', 'a')",
+ testPartialCommit(newArrayList("upsert into " + A_SUCESS_TABLE + " values ('testUpsertFailure1', 'a')",
UPSERT_TO_FAIL,
- "upsert into a_success_table values ('testUpsertFailure2', 'b')"),
+ "upsert into " + A_SUCESS_TABLE + " values ('testUpsertFailure2', 'b')"),
1, new int[]{1}, true,
- newArrayList("select count(*) from a_success_table where k like 'testUpsertFailure_'",
- "select count(*) from " + TABLE_NAME_TO_FAIL + " where k = '" + Bytes.toString(ROW_TO_FAIL) + "'"),
+ newArrayList("select count(*) from " + A_SUCESS_TABLE + " where k like 'testUpsertFailure_'",
+ "select count(*) from " + B_FAILURE_TABLE + " where k = '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "'"),
newArrayList(new Integer(2), new Integer(0)));
}
@Test
public void testUpsertSelectFailure() throws SQLException {
- props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, 100);
-
try (Connection con = driver.connect(url, new Properties())) {
- con.createStatement().execute("upsert into a_success_table values ('" + Bytes.toString(ROW_TO_FAIL) + "', 'boom!')");
+ con.createStatement().execute("upsert into " + A_SUCESS_TABLE + " values ('" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "', 'boom!')");
con.commit();
}
- testPartialCommit(newArrayList("upsert into a_success_table values ('testUpsertSelectFailure', 'a')",
+ testPartialCommit(newArrayList("upsert into " + A_SUCESS_TABLE + " values ('testUpsertSelectFailure', 'a')",
UPSERT_SELECT_TO_FAIL),
1, new int[]{1}, true,
- newArrayList("select count(*) from a_success_table where k in ('testUpsertSelectFailure', '" + Bytes.toString(ROW_TO_FAIL) + "')",
- "select count(*) from " + TABLE_NAME_TO_FAIL + " where k = '" + Bytes.toString(ROW_TO_FAIL) + "'"),
+ newArrayList("select count(*) from " + A_SUCESS_TABLE + " where k in ('testUpsertSelectFailure', '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "')",
+ "select count(*) from " + B_FAILURE_TABLE + " where k = '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "'"),
newArrayList(new Integer(2), new Integer(0)));
}
@Test
public void testDeleteFailure() {
- testPartialCommit(newArrayList("upsert into a_success_table values ('testDeleteFailure1', 'a')",
+ testPartialCommit(newArrayList("upsert into " + A_SUCESS_TABLE + " values ('testDeleteFailure1', 'a')",
DELETE_TO_FAIL,
- "upsert into a_success_table values ('testDeleteFailure2', 'b')"),
+ "upsert into " + A_SUCESS_TABLE + " values ('testDeleteFailure2', 'b')"),
1, new int[]{1}, true,
- newArrayList("select count(*) from a_success_table where k like 'testDeleteFailure_'",
- "select count(*) from " + TABLE_NAME_TO_FAIL + " where k = 'z'"),
+ newArrayList("select count(*) from " + A_SUCESS_TABLE + " where k like 'testDeleteFailure_'",
+ "select count(*) from " + B_FAILURE_TABLE + " where k = 'z'"),
newArrayList(new Integer(2), new Integer(1)));
}
@@ -190,27 +209,27 @@ public class PartialCommitIT {
*/
@Test
public void testOrderOfMutationsIsPredicatable() {
- testPartialCommit(newArrayList("upsert into c_success_table values ('testOrderOfMutationsIsPredicatable', 'c')", // will fail because c_success_table is after b_failure_table by table sort order
+ testPartialCommit(newArrayList("upsert into " + C_SUCESS_TABLE + " values ('testOrderOfMutationsIsPredicatable', 'c')", // will fail because c_success_table is after b_failure_table by table sort order
UPSERT_TO_FAIL,
- "upsert into a_success_table values ('testOrderOfMutationsIsPredicatable', 'a')"), // will succeed because a_success_table is before b_failure_table by table sort order
+ "upsert into " + A_SUCESS_TABLE + " values ('testOrderOfMutationsIsPredicatable', 'a')"), // will succeed because a_success_table is before b_failure_table by table sort order
2, new int[]{0,1}, true,
- newArrayList("select count(*) from c_success_table where k='testOrderOfMutationsIsPredicatable'",
- "select count(*) from a_success_table where k='testOrderOfMutationsIsPredicatable'",
- "select count(*) from " + TABLE_NAME_TO_FAIL + " where k = '" + Bytes.toString(ROW_TO_FAIL) + "'"),
+ newArrayList("select count(*) from " + C_SUCESS_TABLE + " where k='testOrderOfMutationsIsPredicatable'",
+ "select count(*) from " + A_SUCESS_TABLE + " where k='testOrderOfMutationsIsPredicatable'",
+ "select count(*) from " + B_FAILURE_TABLE + " where k = '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "'"),
newArrayList(new Integer(0), new Integer(1), new Integer(0)));
}
@Test
- public void checkThatAllStatementTypesMaintainOrderInConnection() {
- testPartialCommit(newArrayList("upsert into a_success_table values ('k', 'checkThatAllStatementTypesMaintainOrderInConnection')",
- "upsert into a_success_table select k, c from c_success_table",
+ public void testStatementOrderMaintainedInConnection() {
+ testPartialCommit(newArrayList("upsert into " + A_SUCESS_TABLE + " values ('testStatementOrderMaintainedInConnection', 'a')",
+ "upsert into " + A_SUCESS_TABLE + " select k, c from " + C_SUCESS_TABLE,
DELETE_TO_FAIL,
- "select * from a_success_table",
+ "select * from " + A_SUCESS_TABLE + "",
UPSERT_TO_FAIL),
2, new int[]{2,4}, true,
- newArrayList("select count(*) from a_success_table where k='testOrderOfMutationsIsPredicatable' or k like 'z%'", // rows left: zz, zzz, checkThatAllStatementTypesMaintainOrderInConnection
- "select count(*) from " + TABLE_NAME_TO_FAIL + " where k = '" + ROW_TO_FAIL + "'",
- "select count(*) from " + TABLE_NAME_TO_FAIL + " where k = 'z'"),
+ newArrayList("select count(*) from " + A_SUCESS_TABLE + " where k='testStatementOrderMaintainedInConnection' or k like 'z%'", // rows left: zz, zzz, checkThatAllStatementTypesMaintainOrderInConnection
+ "select count(*) from " + B_FAILURE_TABLE + " where k = '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "'",
+ "select count(*) from " + B_FAILURE_TABLE + " where k = 'z'"),
newArrayList(new Integer(4), new Integer(0), new Integer(1)));
}
@@ -259,10 +278,11 @@ public class PartialCommitIT {
Connection con = driver.connect(url, new Properties());
PhoenixConnection phxCon = new PhoenixConnection(con.unwrap(PhoenixConnection.class));
final Map<TableRef,Map<ImmutableBytesPtr,MutationState.RowMutationState>> mutations = Maps.newTreeMap(new TableRefComparator());
- return new PhoenixConnection(phxCon) {
+ // passing a null mutation staate forces the connection.newMutationState() to be used to create the MutationState
+ return new PhoenixConnection(phxCon, null) {
@Override
protected MutationState newMutationState(int maxSize) {
- return new MutationState(maxSize, this, mutations);
+ return new MutationState(maxSize, this, mutations, null);
};
};
}
@@ -271,7 +291,7 @@ public class PartialCommitIT {
@Override
public void prePut(ObserverContext<RegionCoprocessorEnvironment> c, Put put, WALEdit edit,
final Durability durability) throws HBaseIOException {
- if (shouldFailUpsert(c, put)) {
+ if (shouldFail(c, put)) {
// throwing anything other than instances of IOException result
// in this coprocessor being unloaded
// DoNotRetryIOException tells HBase not to retry this mutation
@@ -283,7 +303,7 @@ public class PartialCommitIT {
@Override
public void preDelete(ObserverContext<RegionCoprocessorEnvironment> c,
Delete delete, WALEdit edit, Durability durability) throws IOException {
- if (shouldFailDelete(c, delete)) {
+ if (shouldFail(c, delete)) {
// throwing anything other than instances of IOException result
// in this coprocessor being unloaded
// DoNotRetryIOException tells HBase not to retry this mutation
@@ -292,18 +312,13 @@ public class PartialCommitIT {
}
}
- private boolean shouldFailUpsert(ObserverContext<RegionCoprocessorEnvironment> c, Put put) {
+ private boolean shouldFail(ObserverContext<RegionCoprocessorEnvironment> c, Mutation m) {
String tableName = c.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString();
- return TABLE_NAME_TO_FAIL.equals(tableName) && Bytes.equals(ROW_TO_FAIL, put.getRow());
+ // deletes on transactional tables are converted to put, so use a single helper method
+ return tableName.contains(TABLE_NAME_TO_FAIL) &&
+ (Bytes.equals(ROW_TO_FAIL_UPSERT_BYTES, m.getRow()) || Bytes.equals(ROW_TO_FAIL_DELETE_BYTES, m.getRow()));
}
- private boolean shouldFailDelete(ObserverContext<RegionCoprocessorEnvironment> c, Delete delete) {
- String tableName = c.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString();
- return TABLE_NAME_TO_FAIL.equals(tableName) &&
- // Phoenix deletes are sent as Mutations with empty values
- delete.getFamilyCellMap().firstEntry().getValue().get(0).getValueLength() == 0 &&
- delete.getFamilyCellMap().firstEntry().getValue().get(0).getQualifierLength() == 0;
- }
}
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
index b7f880a..98932e1 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
@@ -20,9 +20,8 @@ package org.apache.phoenix.rpc;
import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA;
import static org.apache.phoenix.util.TestUtil.MUTABLE_INDEX_DATA_TABLE;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-import static org.junit.Assert.assertEquals;
+import static org.apache.phoenix.util.TestUtil.TRANSACTIONAL_DATA_TABLE;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.eq;
import static org.mockito.Matchers.isNull;
@@ -30,33 +29,26 @@ import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
-import java.math.BigDecimal;
import java.sql.Connection;
-import java.sql.Date;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.Map;
import java.util.Properties;
import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
-import org.apache.phoenix.end2end.Shadower;
import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.MetaDataClient;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.types.PVarchar;
-import org.apache.phoenix.util.DateUtil;
+import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
-import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.TestUtil;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
-import com.google.common.collect.Maps;
-
/**
* Verifies the number of rpcs calls from {@link MetaDataClient} updateCache()
* for transactional and non-transactional tables.
@@ -68,86 +60,76 @@ public class UpdateCacheIT extends BaseHBaseManagedTimeIT {
@Before
public void setUp() throws SQLException {
ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE);
+ ensureTableCreated(getUrl(), TRANSACTIONAL_DATA_TABLE);
}
- @BeforeClass
- @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
- public static void doSetup() throws Exception {
- Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
- setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ @Test
+ public void testUpdateCacheForTxnTable() throws Exception {
+ helpTestUpdateCache(true, null);
}
-
- public static void validateRowKeyColumns(ResultSet rs, int i) throws SQLException {
- assertTrue(rs.next());
- assertEquals(rs.getString(1), "varchar" + String.valueOf(i));
- assertEquals(rs.getString(2), "char" + String.valueOf(i));
- assertEquals(rs.getInt(3), i);
- assertEquals(rs.getInt(4), i);
- assertEquals(rs.getBigDecimal(5), new BigDecimal(i*0.5d));
- Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * NUM_MILLIS_IN_DAY);
- assertEquals(rs.getDate(6), date);
- }
-
- public static void setRowKeyColumns(PreparedStatement stmt, int i) throws SQLException {
- // insert row
- stmt.setString(1, "varchar" + String.valueOf(i));
- stmt.setString(2, "char" + String.valueOf(i));
- stmt.setInt(3, i);
- stmt.setLong(4, i);
- stmt.setBigDecimal(5, new BigDecimal(i*0.5d));
- Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * NUM_MILLIS_IN_DAY);
- stmt.setDate(6, date);
+
+ @Test
+ public void testUpdateCacheForNonTxnTable() throws Exception {
+ helpTestUpdateCache(false, null);
}
- @Test
- public void testUpdateCache() throws Exception {
- String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE;
+ public static void helpTestUpdateCache(boolean isTransactional, Long scn) throws Exception {
+ String tableName = isTransactional ? TRANSACTIONAL_DATA_TABLE : MUTABLE_INDEX_DATA_TABLE;
+ String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + tableName;
String selectSql = "SELECT * FROM "+fullTableName;
// use a spyed ConnectionQueryServices so we can verify calls to getTable
ConnectionQueryServices connectionQueryServices = Mockito.spy(driver.getConnectionQueryServices(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)));
Properties props = new Properties();
props.putAll(PhoenixEmbeddedDriver.DEFFAULT_PROPS.asMap());
+ if (scn!=null) {
+ props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(scn));
+ }
Connection conn = connectionQueryServices.connect(getUrl(), props);
try {
conn.setAutoCommit(false);
- ResultSet rs = conn.createStatement().executeQuery(selectSql);
- assertFalse(rs.next());
- reset(connectionQueryServices);
-
String upsert = "UPSERT INTO " + fullTableName + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
PreparedStatement stmt = conn.prepareStatement(upsert);
// upsert three rows
- setRowKeyColumns(stmt, 1);
+ TestUtil.setRowKeyColumns(stmt, 1);
stmt.execute();
- setRowKeyColumns(stmt, 2);
+ TestUtil.setRowKeyColumns(stmt, 2);
stmt.execute();
- setRowKeyColumns(stmt, 3);
+ TestUtil.setRowKeyColumns(stmt, 3);
stmt.execute();
conn.commit();
- // verify only one rpc to getTable occurs after commit is called
- verify(connectionQueryServices, times(1)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(INDEX_DATA_SCHEMA)), eq(PVarchar.INSTANCE.toBytes(MUTABLE_INDEX_DATA_TABLE)), anyLong(), anyLong());
- reset(connectionQueryServices);
+ // verify only one rpc to fetch table metadata,
+ verify(connectionQueryServices).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(INDEX_DATA_SCHEMA)), eq(PVarchar.INSTANCE.toBytes(tableName)), anyLong(), anyLong());
+ reset(connectionQueryServices);
+
+ if (scn!=null) {
+ // advance scn so that we can see the data we just upserted
+ props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(scn+2));
+ conn = connectionQueryServices.connect(getUrl(), props);
+ }
- rs = conn.createStatement().executeQuery(selectSql);
- validateRowKeyColumns(rs, 1);
- validateRowKeyColumns(rs, 2);
- validateRowKeyColumns(rs, 3);
+ ResultSet rs = conn.createStatement().executeQuery(selectSql);
+ TestUtil.validateRowKeyColumns(rs, 1);
+ TestUtil.validateRowKeyColumns(rs, 2);
+ TestUtil.validateRowKeyColumns(rs, 3);
assertFalse(rs.next());
rs = conn.createStatement().executeQuery(selectSql);
- validateRowKeyColumns(rs, 1);
- validateRowKeyColumns(rs, 2);
- validateRowKeyColumns(rs, 3);
+ TestUtil.validateRowKeyColumns(rs, 1);
+ TestUtil.validateRowKeyColumns(rs, 2);
+ TestUtil.validateRowKeyColumns(rs, 3);
assertFalse(rs.next());
rs = conn.createStatement().executeQuery(selectSql);
- validateRowKeyColumns(rs, 1);
- validateRowKeyColumns(rs, 2);
- validateRowKeyColumns(rs, 3);
+ TestUtil.validateRowKeyColumns(rs, 1);
+ TestUtil.validateRowKeyColumns(rs, 2);
+ TestUtil.validateRowKeyColumns(rs, 3);
assertFalse(rs.next());
- conn.commit();
- // there should be one rpc to getTable per query
- verify(connectionQueryServices, times(3)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(INDEX_DATA_SCHEMA)), eq(PVarchar.INSTANCE.toBytes(MUTABLE_INDEX_DATA_TABLE)), anyLong(), anyLong());
+
+ // for non-transactional tables without a scn : verify one rpc to getTable occurs *per* query
+ // for non-transactional tables with a scn : verify *only* one rpc occurs
+ // for transactional tables : verify *only* one rpc occurs
+ int numRpcs = isTransactional || scn!=null ? 1 : 3;
+ verify(connectionQueryServices, times(numRpcs)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(INDEX_DATA_SCHEMA)), eq(PVarchar.INSTANCE.toBytes(tableName)), anyLong(), anyLong());
}
finally {
conn.close();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java
new file mode 100644
index 0000000..dbc7fd1
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.rpc;
+
+import static org.apache.phoenix.util.TestUtil.MUTABLE_INDEX_DATA_TABLE;
+
+import org.apache.phoenix.end2end.BaseClientManagedTimeIT;
+import org.junit.Before;
+import org.junit.Test;
+
+public class UpdateCacheWithScnIT extends BaseClientManagedTimeIT {
+
+ protected long ts;
+
+ @Before
+ public void initTable() throws Exception {
+ ts = nextTimestamp();
+ ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE, ts);
+ }
+
+ @Test
+ public void testUpdateCacheWithScn() throws Exception {
+ UpdateCacheIT.helpTestUpdateCache(false, ts+2);
+ }
+
+}
[15/52] [abbrv] phoenix git commit: PHOENIX-2449 QueryServer needs
Hadoop configuration on classpath with Kerberos(Josh Elser)
Posted by ma...@apache.org.
PHOENIX-2449 QueryServer needs Hadoop configuration on classpath with Kerberos(Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/cab39d72
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/cab39d72
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/cab39d72
Branch: refs/heads/calcite
Commit: cab39d72171ff76397661fec93d55da7420352fb
Parents: 1b5e7fd
Author: Enis Soztutar <en...@apache.org>
Authored: Wed Nov 25 10:55:00 2015 -0800
Committer: Enis Soztutar <en...@apache.org>
Committed: Wed Nov 25 10:55:00 2015 -0800
----------------------------------------------------------------------
bin/phoenix_utils.py | 15 +++++++++++++--
bin/queryserver.py | 4 +++-
2 files changed, 16 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cab39d72/bin/phoenix_utils.py
----------------------------------------------------------------------
diff --git a/bin/phoenix_utils.py b/bin/phoenix_utils.py
index 068d097..5fb4f08 100755
--- a/bin/phoenix_utils.py
+++ b/bin/phoenix_utils.py
@@ -85,8 +85,11 @@ def setPath():
# else fall back to HBASE_HOME
if os.getenv('HBASE_HOME'):
hbase_conf_dir = os.path.join(os.getenv('HBASE_HOME'), "conf")
+ elif os.name == 'posix':
+ # default to the bigtop configuration dir
+ hbase_conf_dir = '/etc/hbase/conf'
else:
- # default to pwd
+ # Try to provide something valid
hbase_conf_dir = '.'
global hbase_conf_path # keep conf_path around for backward compatibility
hbase_conf_path = hbase_conf_dir
@@ -114,7 +117,15 @@ def setPath():
phoenix_test_jar_path = os.path.join(current_dir, "..", "phoenix-core", "target","*")
global hadoop_conf
- hadoop_conf = os.getenv('HADOOP_CONF_DIR', '')
+ hadoop_conf = os.getenv('HADOOP_CONF_DIR', None)
+ if not hadoop_conf:
+ if os.name == 'posix':
+ # Try to provide a sane configuration directory for Hadoop if not otherwise provided.
+ # If there's no jaas file specified by the caller, this is necessary when Kerberos is enabled.
+ hadoop_conf = '/etc/hadoop/conf'
+ else:
+ # Try to provide something valid..
+ hadoop_conf = '.'
global hadoop_classpath
if (os.name != 'nt'):
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cab39d72/bin/queryserver.py
----------------------------------------------------------------------
diff --git a/bin/queryserver.py b/bin/queryserver.py
index 11f5aeb..78d340c 100755
--- a/bin/queryserver.py
+++ b/bin/queryserver.py
@@ -67,6 +67,7 @@ else:
# HBase configuration folder path (where hbase-site.xml reside) for
# HBase/Phoenix client side property override
hbase_config_path = phoenix_utils.hbase_conf_dir
+hadoop_config_path = phoenix_utils.hadoop_conf
# default paths ## TODO: add windows support
java_home = os.getenv('JAVA_HOME')
@@ -118,7 +119,8 @@ else:
# " -Xdebug -Xrunjdwp:transport=dt_socket,address=5005,server=y,suspend=n " + \
# " -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true" + \
-java_cmd = '%(java)s -cp ' + hbase_config_path + os.pathsep + phoenix_utils.phoenix_queryserver_jar + os.pathsep + phoenix_utils.phoenix_client_jar + \
+java_cmd = '%(java)s -cp ' + hbase_config_path + os.pathsep + hadoop_config_path + os.pathsep + \
+ phoenix_utils.phoenix_queryserver_jar + os.pathsep + phoenix_utils.phoenix_client_jar + \
" -Dproc_phoenixserver" + \
" -Dlog4j.configuration=file:" + os.path.join(phoenix_utils.current_dir, "log4j.properties") + \
" -Dpsql.root.logger=%(root_logger)s" + \
[28/52] [abbrv] phoenix git commit: PHOENIX-2466 Correct warning on
maven-dependency-plugin in phoenix-core (Gabor Liptak)
Posted by ma...@apache.org.
PHOENIX-2466 Correct warning on maven-dependency-plugin in phoenix-core (Gabor Liptak)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d4a4376e
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d4a4376e
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d4a4376e
Branch: refs/heads/calcite
Commit: d4a4376eb8daec8007a6cc257fdfeeee778762cf
Parents: 21e7c7d
Author: James Taylor <jt...@salesforce.com>
Authored: Mon Nov 30 14:21:16 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Nov 30 14:21:16 2015 -0800
----------------------------------------------------------------------
phoenix-core/pom.xml | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d4a4376e/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index d003e8a..8fe7ecc 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -202,11 +202,9 @@
<artifactId>maven-failsafe-plugin</artifactId>
</plugin>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>${maven-dependency-plugin.version}</version>
- </plugin>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<!-- generates the file that will be used by the sandbox script in the dev env -->
[48/52] [abbrv] phoenix git commit: PHOENIX-2494 Fix test failure
AlterTableWithViewsIT.testMakeBaseTableTransactional
Posted by ma...@apache.org.
PHOENIX-2494 Fix test failure AlterTableWithViewsIT.testMakeBaseTableTransactional
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/bf066177
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/bf066177
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/bf066177
Branch: refs/heads/calcite
Commit: bf06617754ac408279a1359cc660239fbab05ad5
Parents: b74215f
Author: Thomas D'Silva <td...@salesforce.com>
Authored: Tue Dec 8 16:51:26 2015 -0800
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Tue Dec 8 16:51:26 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/bf066177/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index f997149..4361b1d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -1838,7 +1838,6 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
}
// Sort the puts by ordinal position
Collections.sort(columnPutsForBaseTable);
- assert columnPutsForBaseTable.size() > 0;
for (Result viewResult : childViewsResult.getResults()) {
short deltaNumPkColsSoFar = 0;
short columnsAddedToView = 0;
[07/52] [abbrv] phoenix git commit: PHOENIX-1674 Snapshot isolation
transaction support through Tephra (James Taylor, Thomas D'Silva)
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
new file mode 100644
index 0000000..6bcf7ea
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
@@ -0,0 +1,537 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.tx;
+
+import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.apache.phoenix.util.TestUtil.TRANSACTIONAL_DATA_TABLE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.Before;
+import org.junit.Test;
+
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.hbase11.coprocessor.TransactionProcessor;
+
+import com.google.common.collect.Lists;
+
+public class TransactionIT extends BaseHBaseManagedTimeIT {
+
+ private static final String FULL_TABLE_NAME = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + TRANSACTIONAL_DATA_TABLE;
+
+ @Before
+ public void setUp() throws SQLException {
+ ensureTableCreated(getUrl(), TRANSACTIONAL_DATA_TABLE);
+ }
+
+ @Test
+ public void testReadOwnWrites() throws Exception {
+ String selectSql = "SELECT * FROM "+FULL_TABLE_NAME;
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(false);
+ ResultSet rs = conn.createStatement().executeQuery(selectSql);
+ assertFalse(rs.next());
+
+ String upsert = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn.prepareStatement(upsert);
+ // upsert two rows
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.execute();
+ TestUtil.setRowKeyColumns(stmt, 2);
+ stmt.execute();
+
+ // verify rows can be read even though commit has not been called
+ rs = conn.createStatement().executeQuery(selectSql);
+ TestUtil.validateRowKeyColumns(rs, 1);
+ TestUtil.validateRowKeyColumns(rs, 2);
+ assertFalse(rs.next());
+
+ conn.commit();
+
+ // verify rows can be read after commit
+ rs = conn.createStatement().executeQuery(selectSql);
+ TestUtil.validateRowKeyColumns(rs, 1);
+ TestUtil.validateRowKeyColumns(rs, 2);
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testTxnClosedCorrecty() throws Exception {
+ String selectSql = "SELECT * FROM "+FULL_TABLE_NAME;
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(false);
+ ResultSet rs = conn.createStatement().executeQuery(selectSql);
+ assertFalse(rs.next());
+
+ String upsert = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn.prepareStatement(upsert);
+ // upsert two rows
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.execute();
+ TestUtil.setRowKeyColumns(stmt, 2);
+ stmt.execute();
+
+ // verify rows can be read even though commit has not been called
+ rs = conn.createStatement().executeQuery(selectSql);
+ TestUtil.validateRowKeyColumns(rs, 1);
+ TestUtil.validateRowKeyColumns(rs, 2);
+ assertFalse(rs.next());
+
+ conn.close();
+ // wait for any open txns to time out
+ Thread.sleep(DEFAULT_TXN_TIMEOUT_SECONDS*1000+10000);
+ assertTrue("There should be no invalid transactions", txManager.getInvalidSize()==0);
+ }
+ }
+
+ @Test
+ public void testDelete() throws Exception {
+ String selectSQL = "SELECT * FROM " + FULL_TABLE_NAME;
+ try (Connection conn1 = DriverManager.getConnection(getUrl());
+ Connection conn2 = DriverManager.getConnection(getUrl())) {
+ conn1.setAutoCommit(false);
+ ResultSet rs = conn1.createStatement().executeQuery(selectSQL);
+ assertFalse(rs.next());
+
+ String upsert = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn1.prepareStatement(upsert);
+ // upsert two rows
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.execute();
+ conn1.commit();
+
+ TestUtil.setRowKeyColumns(stmt, 2);
+ stmt.execute();
+
+ // verify rows can be read even though commit has not been called
+ int rowsDeleted = conn1.createStatement().executeUpdate("DELETE FROM " + FULL_TABLE_NAME);
+ assertEquals(2, rowsDeleted);
+
+ // Delete and second upsert not committed yet, so there should be one row.
+ rs = conn2.createStatement().executeQuery("SELECT count(*) FROM " + FULL_TABLE_NAME);
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+
+ conn1.commit();
+
+ // verify rows are deleted after commit
+ rs = conn1.createStatement().executeQuery(selectSQL);
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testAutoCommitQuerySingleTable() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(true);
+ // verify no rows returned
+ ResultSet rs = conn.createStatement().executeQuery("SELECT * FROM " + FULL_TABLE_NAME);
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testAutoCommitQueryMultiTables() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(true);
+ // verify no rows returned
+ ResultSet rs = conn.createStatement().executeQuery("SELECT * FROM " + FULL_TABLE_NAME + " a JOIN " + FULL_TABLE_NAME + " b ON (a.long_pk = b.int_pk)");
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testColConflicts() throws Exception {
+ try (Connection conn1 = DriverManager.getConnection(getUrl());
+ Connection conn2 = DriverManager.getConnection(getUrl())) {
+ conn1.setAutoCommit(false);
+ conn2.setAutoCommit(false);
+ String selectSql = "SELECT * FROM "+FULL_TABLE_NAME;
+ conn1.setAutoCommit(false);
+ ResultSet rs = conn1.createStatement().executeQuery(selectSql);
+ assertFalse(rs.next());
+ // upsert row using conn1
+ String upsertSql = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk, a.int_col1) VALUES(?, ?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn1.prepareStatement(upsertSql);
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.setInt(7, 10);
+ stmt.execute();
+ // upsert row using conn2
+ stmt = conn2.prepareStatement(upsertSql);
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.setInt(7, 11);
+ stmt.execute();
+
+ conn1.commit();
+ //second commit should fail
+ try {
+ conn2.commit();
+ fail();
+ }
+ catch (SQLException e) {
+ assertEquals(e.getErrorCode(), SQLExceptionCode.TRANSACTION_CONFLICT_EXCEPTION.getErrorCode());
+ }
+ }
+ }
+
+ private void testRowConflicts() throws Exception {
+ try (Connection conn1 = DriverManager.getConnection(getUrl());
+ Connection conn2 = DriverManager.getConnection(getUrl())) {
+ conn1.setAutoCommit(false);
+ conn2.setAutoCommit(false);
+ String selectSql = "SELECT * FROM "+FULL_TABLE_NAME;
+ conn1.setAutoCommit(false);
+ ResultSet rs = conn1.createStatement().executeQuery(selectSql);
+ boolean immutableRows = conn1.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, FULL_TABLE_NAME)).isImmutableRows();
+ assertFalse(rs.next());
+ // upsert row using conn1
+ String upsertSql = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk, a.int_col1) VALUES(?, ?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn1.prepareStatement(upsertSql);
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.setInt(7, 10);
+ stmt.execute();
+ // upsert row using conn2
+ upsertSql = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk, b.int_col2) VALUES(?, ?, ?, ?, ?, ?, ?)";
+ stmt = conn2.prepareStatement(upsertSql);
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.setInt(7, 11);
+ stmt.execute();
+
+ conn1.commit();
+ //second commit should fail
+ try {
+ conn2.commit();
+ if (!immutableRows) fail();
+ }
+ catch (SQLException e) {
+ if (immutableRows) fail();
+ assertEquals(e.getErrorCode(), SQLExceptionCode.TRANSACTION_CONFLICT_EXCEPTION.getErrorCode());
+ }
+ }
+ }
+
+ @Test
+ public void testRowConflictDetected() throws Exception {
+ testRowConflicts();
+ }
+
+ @Test
+ public void testNoConflictDetectionForImmutableRows() throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ conn.createStatement().execute("ALTER TABLE " + FULL_TABLE_NAME + " SET IMMUTABLE_ROWS=true");
+ testRowConflicts();
+ }
+
+ @Test
+ public void testNonTxToTxTable() throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ conn.createStatement().execute("CREATE TABLE NON_TX_TABLE(k INTEGER PRIMARY KEY, v VARCHAR)");
+ conn.createStatement().execute("UPSERT INTO NON_TX_TABLE VALUES (1)");
+ conn.createStatement().execute("UPSERT INTO NON_TX_TABLE VALUES (2, 'a')");
+ conn.createStatement().execute("UPSERT INTO NON_TX_TABLE VALUES (3, 'b')");
+ conn.commit();
+
+ conn.createStatement().execute("CREATE INDEX IDX ON NON_TX_TABLE(v)");
+ // Reset empty column value to an empty value like it is pre-transactions
+ HTableInterface htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("NON_TX_TABLE"));
+ List<Put>puts = Lists.newArrayList(new Put(PInteger.INSTANCE.toBytes(1)), new Put(PInteger.INSTANCE.toBytes(2)), new Put(PInteger.INSTANCE.toBytes(3)));
+ for (Put put : puts) {
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, ByteUtil.EMPTY_BYTE_ARRAY);
+ }
+ htable.put(puts);
+
+ conn.createStatement().execute("ALTER TABLE NON_TX_TABLE SET TRANSACTIONAL=true");
+
+ htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("NON_TX_TABLE"));
+ assertTrue(htable.getTableDescriptor().getCoprocessors().contains(TransactionProcessor.class.getName()));
+ htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("IDX"));
+ assertTrue(htable.getTableDescriptor().getCoprocessors().contains(TransactionProcessor.class.getName()));
+
+ conn.createStatement().execute("UPSERT INTO NON_TX_TABLE VALUES (4, 'c')");
+ ResultSet rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ k FROM NON_TX_TABLE WHERE v IS NULL");
+ assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "NON_TX_TABLE")).isTransactional());
+ assertTrue(rs.next());
+ assertEquals(1,rs.getInt(1));
+ assertFalse(rs.next());
+ conn.commit();
+
+ conn.createStatement().execute("UPSERT INTO NON_TX_TABLE VALUES (5, 'd')");
+ rs = conn.createStatement().executeQuery("SELECT k FROM NON_TX_TABLE");
+ assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "IDX")).isTransactional());
+ assertTrue(rs.next());
+ assertEquals(1,rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(2,rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(3,rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(4,rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(5,rs.getInt(1));
+ assertFalse(rs.next());
+ conn.rollback();
+
+ rs = conn.createStatement().executeQuery("SELECT k FROM NON_TX_TABLE");
+ assertTrue(rs.next());
+ assertEquals(1,rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(2,rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(3,rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(4,rs.getInt(1));
+ assertFalse(rs.next());
+ }
+
+ @Test
+ public void testNonTxToTxTableFailure() throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ // Put table in SYSTEM schema to prevent attempts to update the cache after we disable SYSTEM.CATALOG
+ conn.createStatement().execute("CREATE TABLE SYSTEM.NON_TX_TABLE(k INTEGER PRIMARY KEY, v VARCHAR)");
+ conn.createStatement().execute("UPSERT INTO SYSTEM.NON_TX_TABLE VALUES (1)");
+ conn.commit();
+ // Reset empty column value to an empty value like it is pre-transactions
+ HTableInterface htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("SYSTEM.NON_TX_TABLE"));
+ Put put = new Put(PInteger.INSTANCE.toBytes(1));
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, ByteUtil.EMPTY_BYTE_ARRAY);
+ htable.put(put);
+
+ HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
+ admin.disableTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME);
+ try {
+ // This will succeed initially in updating the HBase metadata, but then will fail when
+ // the SYSTEM.CATALOG table is attempted to be updated, exercising the code to restore
+ // the coprocessors back to the non transactional ones.
+ conn.createStatement().execute("ALTER TABLE SYSTEM.NON_TX_TABLE SET TRANSACTIONAL=true");
+ fail();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME + " is disabled"));
+ } finally {
+ admin.enableTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME);
+ admin.close();
+ }
+
+ ResultSet rs = conn.createStatement().executeQuery("SELECT k FROM SYSTEM.NON_TX_TABLE WHERE v IS NULL");
+ assertTrue(rs.next());
+ assertEquals(1,rs.getInt(1));
+ assertFalse(rs.next());
+
+ htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("SYSTEM.NON_TX_TABLE"));
+ assertFalse(htable.getTableDescriptor().getCoprocessors().contains(TransactionProcessor.class.getName()));
+ assertEquals(1,conn.unwrap(PhoenixConnection.class).getQueryServices().
+ getTableDescriptor(Bytes.toBytes("SYSTEM.NON_TX_TABLE")).
+ getFamily(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES).getMaxVersions());
+ }
+
+ @Test
+ public void testProperties() throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ conn.createStatement().execute("CREATE TABLE NON_TX_TABLE1(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR) TTL=1000");
+ conn.createStatement().execute("CREATE INDEX idx1 ON NON_TX_TABLE1(a.v, b.v) TTL=1000");
+ conn.createStatement().execute("CREATE INDEX idx2 ON NON_TX_TABLE1(c.v) INCLUDE (a.v, b.v) TTL=1000");
+
+ conn.createStatement().execute("ALTER TABLE NON_TX_TABLE1 SET TRANSACTIONAL=true");
+
+ HTableDescriptor desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("NON_TX_TABLE1"));
+ for (HColumnDescriptor colDesc : desc.getFamilies()) {
+ assertEquals(QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
+ assertEquals(1000, colDesc.getTimeToLive());
+ assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
+ }
+
+ desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("IDX1"));
+ for (HColumnDescriptor colDesc : desc.getFamilies()) {
+ assertEquals(QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
+ assertEquals(1000, colDesc.getTimeToLive());
+ assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
+ }
+
+ desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("IDX2"));
+ for (HColumnDescriptor colDesc : desc.getFamilies()) {
+ assertEquals(QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
+ assertEquals(1000, colDesc.getTimeToLive());
+ assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
+ }
+
+ conn.createStatement().execute("CREATE TABLE NON_TX_TABLE2(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)");
+ conn.createStatement().execute("ALTER TABLE NON_TX_TABLE2 SET TRANSACTIONAL=true, VERSIONS=10");
+ desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("NON_TX_TABLE2"));
+ for (HColumnDescriptor colDesc : desc.getFamilies()) {
+ assertEquals(10, colDesc.getMaxVersions());
+ assertEquals(HColumnDescriptor.DEFAULT_TTL, colDesc.getTimeToLive());
+ assertEquals(null, colDesc.getValue(TxConstants.PROPERTY_TTL));
+ }
+ conn.createStatement().execute("ALTER TABLE NON_TX_TABLE2 SET TTL=1000");
+ desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("NON_TX_TABLE2"));
+ for (HColumnDescriptor colDesc : desc.getFamilies()) {
+ assertEquals(10, colDesc.getMaxVersions());
+ assertEquals(1000, colDesc.getTimeToLive());
+ assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
+ }
+
+ conn.createStatement().execute("CREATE TABLE NON_TX_TABLE3(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)");
+ conn.createStatement().execute("ALTER TABLE NON_TX_TABLE3 SET TRANSACTIONAL=true, b.VERSIONS=10, c.VERSIONS=20");
+ desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("NON_TX_TABLE3"));
+ assertEquals(QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, desc.getFamily(Bytes.toBytes("A")).getMaxVersions());
+ assertEquals(10, desc.getFamily(Bytes.toBytes("B")).getMaxVersions());
+ assertEquals(20, desc.getFamily(Bytes.toBytes("C")).getMaxVersions());
+
+ conn.createStatement().execute("CREATE TABLE NON_TX_TABLE4(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)");
+ try {
+ conn.createStatement().execute("ALTER TABLE NON_TX_TABLE4 SET TRANSACTIONAL=true, VERSIONS=1");
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE.getErrorCode(), e.getErrorCode());
+ }
+
+ try {
+ conn.createStatement().execute("ALTER TABLE NON_TX_TABLE4 SET TRANSACTIONAL=true, b.VERSIONS=1");
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE.getErrorCode(), e.getErrorCode());
+ }
+
+ conn.createStatement().execute("CREATE TABLE TX_TABLE1(k INTEGER PRIMARY KEY, v VARCHAR) TTL=1000, TRANSACTIONAL=true");
+ desc = conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(Bytes.toBytes("TX_TABLE1"));
+ for (HColumnDescriptor colDesc : desc.getFamilies()) {
+ assertEquals(QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
+ assertEquals(HColumnDescriptor.DEFAULT_TTL, colDesc.getTimeToLive());
+ assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
+ }
+ }
+
+ @Test
+ public void testCreateTableToBeTransactional() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String ddl = "CREATE TABLE TEST_TRANSACTIONAL_TABLE (k varchar primary key) transactional=true";
+ conn.createStatement().execute(ddl);
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ PTable table = pconn.getTable(new PTableKey(null, "TEST_TRANSACTIONAL_TABLE"));
+ HTableInterface htable = pconn.getQueryServices().getTable(Bytes.toBytes("TEST_TRANSACTIONAL_TABLE"));
+ assertTrue(table.isTransactional());
+ assertTrue(htable.getTableDescriptor().getCoprocessors().contains(TransactionProcessor.class.getName()));
+
+ try {
+ ddl = "ALTER TABLE TEST_TRANSACTIONAL_TABLE SET transactional=false";
+ conn.createStatement().execute(ddl);
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.TX_MAY_NOT_SWITCH_TO_NON_TX.getErrorCode(), e.getErrorCode());
+ }
+
+ HBaseAdmin admin = pconn.getQueryServices().getAdmin();
+ HTableDescriptor desc = new HTableDescriptor(TableName.valueOf("TXN_TEST_EXISTING"));
+ desc.addFamily(new HColumnDescriptor(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES));
+ admin.createTable(desc);
+ ddl = "CREATE TABLE TXN_TEST_EXISTING (k varchar primary key) transactional=true";
+ conn.createStatement().execute(ddl);
+ assertEquals(Boolean.TRUE.toString(), admin.getTableDescriptor(TableName.valueOf("TXN_TEST_EXISTING")).getValue(TxConstants.READ_NON_TX_DATA));
+
+ // Should be ok, as HBase metadata should match existing metadata.
+ ddl = "CREATE TABLE IF NOT EXISTS TEST_TRANSACTIONAL_TABLE (k varchar primary key)";
+ try {
+ conn.createStatement().execute(ddl);
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.TX_MAY_NOT_SWITCH_TO_NON_TX.getErrorCode(), e.getErrorCode());
+ }
+ ddl += " transactional=true";
+ conn.createStatement().execute(ddl);
+ table = pconn.getTable(new PTableKey(null, "TEST_TRANSACTIONAL_TABLE"));
+ htable = pconn.getQueryServices().getTable(Bytes.toBytes("TEST_TRANSACTIONAL_TABLE"));
+ assertTrue(table.isTransactional());
+ assertTrue(htable.getTableDescriptor().getCoprocessors().contains(TransactionProcessor.class.getName()));
+ }
+
+ public void testCurrentDate() throws Exception {
+ String selectSql = "SELECT current_date() FROM "+FULL_TABLE_NAME;
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(false);
+ ResultSet rs = conn.createStatement().executeQuery(selectSql);
+ assertFalse(rs.next());
+
+ String upsert = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn.prepareStatement(upsert);
+ // upsert two rows
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.execute();
+ conn.commit();
+
+ rs = conn.createStatement().executeQuery(selectSql);
+ assertTrue(rs.next());
+ Date date1 = rs.getDate(1);
+ assertFalse(rs.next());
+
+ Thread.sleep(1000);
+
+ rs = conn.createStatement().executeQuery(selectSql);
+ assertTrue(rs.next());
+ Date date2 = rs.getDate(1);
+ assertFalse(rs.next());
+ assertTrue("current_date() should change while executing multiple statements", date2.getTime() > date1.getTime());
+ }
+ }
+
+ @Test
+ public void testReCreateTxnTableAfterDroppingExistingNonTxnTable() throws SQLException {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE DEMO(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
+ stmt.execute("DROP TABLE DEMO");
+ stmt.execute("CREATE TABLE DEMO(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) TRANSACTIONAL=true");
+ stmt.execute("CREATE INDEX DEMO_IDX ON DEMO (v1) INCLUDE(v2)");
+ assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "DEMO")).isTransactional());
+ assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "DEMO_IDX")).isTransactional());
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java
new file mode 100644
index 0000000..34869dd
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.tx;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.end2end.Shadower;
+import org.apache.phoenix.execute.MutationState;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import co.cask.tephra.Transaction.VisibilityLevel;
+
+import com.google.common.collect.Maps;
+
+@RunWith(Parameterized.class)
+public class TxCheckpointIT extends BaseHBaseManagedTimeIT {
+
+ private final boolean localIndex;
+ private final boolean mutable;
+ private String tableName;
+ private String indexName;
+ private String seqName;
+ private String fullTableName;
+
+ public TxCheckpointIT(boolean localIndex, boolean mutable) {
+ this.localIndex = localIndex;
+ this.mutable = mutable;
+ this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME;
+ this.indexName = "IDX_" + System.currentTimeMillis();
+ this.seqName = "SEQ_" + System.currentTimeMillis();
+ this.fullTableName = SchemaUtil.getTableName(tableName, tableName);
+ }
+
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(1);
+ props.put(QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @Parameters(name="localIndex = {0} , mutable = {1}")
+ public static Collection<Boolean[]> data() {
+ return Arrays.asList(new Boolean[][] {
+ { false, false }, { false, true }, { true, false }, { true, true }
+ });
+ }
+
+ @Test
+ public void testUpsertSelectDoesntSeeUpsertedData() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(QueryServices.MUTATE_BATCH_SIZE_ATTRIB, Integer.toString(3));
+ props.setProperty(QueryServices.SCAN_CACHE_SIZE_ATTRIB, Integer.toString(3));
+ props.setProperty(QueryServices.SCAN_RESULT_CHUNK_SIZE, Integer.toString(3));
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(true);
+ conn.createStatement().execute("CREATE SEQUENCE "+seqName);
+ conn.createStatement().execute("CREATE TABLE " + fullTableName + "(pk INTEGER PRIMARY KEY, val INTEGER)"+(!mutable? " IMMUTABLE_ROWS=true" : ""));
+ conn.createStatement().execute("CREATE "+(localIndex? "LOCAL " : "")+"INDEX " + indexName + " ON " + fullTableName + "(val)");
+
+ conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES (NEXT VALUE FOR " + seqName + ",1)");
+ for (int i=0; i<6; i++) {
+ Statement stmt = conn.createStatement();
+ int upsertCount = stmt.executeUpdate("UPSERT INTO " + fullTableName + " SELECT NEXT VALUE FOR " + seqName + ", val FROM " + fullTableName);
+ assertEquals((int)Math.pow(2, i), upsertCount);
+ }
+ conn.close();
+ }
+
+ @Test
+ public void testRollbackOfUncommittedDelete() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"+(!mutable? " IMMUTABLE_ROWS=true" : ""));
+ stmt.execute("CREATE "+(localIndex? "LOCAL " : "")+"INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE(v2)");
+
+ stmt.executeUpdate("upsert into " + fullTableName + " values('x1', 'y1', 'a1')");
+ stmt.executeUpdate("upsert into " + fullTableName + " values('x2', 'y2', 'a2')");
+
+ //assert values in data table
+ ResultSet rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName + " ORDER BY k");
+ assertTrue(rs.next());
+ assertEquals("x1", rs.getString(1));
+ assertEquals("y1", rs.getString(2));
+ assertEquals("a1", rs.getString(3));
+ assertTrue(rs.next());
+ assertEquals("x2", rs.getString(1));
+ assertEquals("y2", rs.getString(2));
+ assertEquals("a2", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert values in index table
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x1", rs.getString(1));
+ assertEquals("y1", rs.getString(2));
+ assertEquals("a1", rs.getString(3));
+ assertTrue(rs.next());
+ assertEquals("x2", rs.getString(1));
+ assertEquals("y2", rs.getString(2));
+ assertEquals("a2", rs.getString(3));
+ assertFalse(rs.next());
+
+ conn.commit();
+
+ stmt.executeUpdate("DELETE FROM " + fullTableName + " WHERE k='x1' AND v1='y1' AND v2='a1'");
+ //assert row is delete in data table
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName + " ORDER BY k");
+ assertTrue(rs.next());
+ assertEquals("x2", rs.getString(1));
+ assertEquals("y2", rs.getString(2));
+ assertEquals("a2", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert row is delete in index table
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x2", rs.getString(1));
+ assertEquals("y2", rs.getString(2));
+ assertEquals("a2", rs.getString(3));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ //assert two rows in data table
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName + " ORDER BY k");
+ assertTrue(rs.next());
+ assertEquals("x1", rs.getString(1));
+ assertEquals("y1", rs.getString(2));
+ assertEquals("a1", rs.getString(3));
+ assertTrue(rs.next());
+ assertEquals("x2", rs.getString(1));
+ assertEquals("y2", rs.getString(2));
+ assertEquals("a2", rs.getString(3));
+ assertFalse(rs.next());
+
+ //assert two rows in index table
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName + " ORDER BY v1");
+ assertTrue(rs.next());
+ assertEquals("x1", rs.getString(1));
+ assertEquals("y1", rs.getString(2));
+ assertEquals("a1", rs.getString(3));
+ assertTrue(rs.next());
+ assertEquals("x2", rs.getString(1));
+ assertEquals("y2", rs.getString(2));
+ assertEquals("a2", rs.getString(3));
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testCheckpointForUpsertSelect() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);) {
+ conn.setAutoCommit(false);
+ Statement stmt = conn.createStatement();
+
+ stmt.execute("CREATE TABLE " + fullTableName + "(ID BIGINT NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"
+ + (!mutable ? " IMMUTABLE_ROWS=true" : ""));
+ stmt.execute("CREATE " + (localIndex ? "LOCAL " : "")
+ + "INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE(v2)");
+
+ stmt.executeUpdate("upsert into " + fullTableName + " values(1, 'a2', 'b1')");
+ stmt.executeUpdate("upsert into " + fullTableName + " values(2, 'a2', 'b2')");
+ stmt.executeUpdate("upsert into " + fullTableName + " values(3, 'a3', 'b3')");
+ conn.commit();
+
+ upsertRows(conn);
+ conn.rollback();
+ verifyRows(conn, 3);
+
+ upsertRows(conn);
+ conn.commit();
+ verifyRows(conn, 6);
+ }
+ }
+
+ private void verifyRows(Connection conn, int expectedMaxId) throws SQLException {
+ ResultSet rs;
+ //query the data table
+ rs = conn.createStatement().executeQuery("select /*+ NO_INDEX */ max(id) from " + fullTableName + "");
+ assertTrue(rs.next());
+ assertEquals(expectedMaxId, rs.getLong(1));
+ assertFalse(rs.next());
+
+ // query the index
+ rs = conn.createStatement().executeQuery("select /*+ INDEX(DEMO IDX) */ max(id) from " + fullTableName + "");
+ assertTrue(rs.next());
+ assertEquals(expectedMaxId, rs.getLong(1));
+ assertFalse(rs.next());
+ }
+
+ private void upsertRows(Connection conn) throws SQLException {
+ ResultSet rs;
+ MutationState state = conn.unwrap(PhoenixConnection.class)
+ .getMutationState();
+ state.startTransaction();
+ long wp = state.getWritePointer();
+ conn.createStatement().execute(
+ "upsert into " + fullTableName + " select max(id)+1, 'a4', 'b4' from " + fullTableName + "");
+ assertEquals(VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT,
+ state.getVisibilityLevel());
+ assertEquals(wp, state.getWritePointer()); // Make sure write ptr
+ // didn't move
+ rs = conn.createStatement().executeQuery("select max(id) from " + fullTableName + "");
+
+ assertTrue(rs.next());
+ assertEquals(4, rs.getLong(1));
+ assertFalse(rs.next());
+
+ conn.createStatement().execute(
+ "upsert into " + fullTableName + " select max(id)+1, 'a5', 'b5' from " + fullTableName + "");
+ assertEquals(VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT,
+ state.getVisibilityLevel());
+ assertNotEquals(wp, state.getWritePointer()); // Make sure write ptr
+ // moves
+ wp = state.getWritePointer();
+ rs = conn.createStatement().executeQuery("select max(id) from " + fullTableName + "");
+
+ assertTrue(rs.next());
+ assertEquals(5, rs.getLong(1));
+ assertFalse(rs.next());
+
+ conn.createStatement().execute(
+ "upsert into " + fullTableName + " select max(id)+1, 'a6', 'b6' from " + fullTableName + "");
+ assertEquals(VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT,
+ state.getVisibilityLevel());
+ assertNotEquals(wp, state.getWritePointer()); // Make sure write ptr
+ // moves
+ wp = state.getWritePointer();
+ rs = conn.createStatement().executeQuery("select max(id) from " + fullTableName + "");
+
+ assertTrue(rs.next());
+ assertEquals(6, rs.getLong(1));
+ assertFalse(rs.next());
+ }
+
+ @Test
+ public void testCheckpointForDeleteAndUpsert() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ ResultSet rs;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);) {
+ conn.setAutoCommit(false);
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName + "1(ID1 BIGINT NOT NULL PRIMARY KEY, FK1A INTEGER, FK1B INTEGER)"
+ + (!mutable ? " IMMUTABLE_ROWS=true" : ""));
+ stmt.execute("CREATE TABLE " + fullTableName + "2(ID2 BIGINT NOT NULL PRIMARY KEY, FK2 INTEGER)"
+ + (!mutable ? " IMMUTABLE_ROWS=true" : ""));
+ stmt.execute("CREATE " + (localIndex ? "LOCAL " : "")
+ + "INDEX " + indexName + " ON " + fullTableName + "1 (FK1B)");
+
+ stmt.executeUpdate("upsert into " + fullTableName + "1 values (1, 3, 3)");
+ stmt.executeUpdate("upsert into " + fullTableName + "1 values (2, 2, 2)");
+ stmt.executeUpdate("upsert into " + fullTableName + "1 values (3, 1, 1)");
+ stmt.executeUpdate("upsert into " + fullTableName + "2 values (1, 1)");
+ conn.commit();
+
+ MutationState state = conn.unwrap(PhoenixConnection.class).getMutationState();
+ state.startTransaction();
+ long wp = state.getWritePointer();
+ conn.createStatement().execute("delete from " + fullTableName + "1 where id1=fk1b AND fk1b=id1");
+ assertEquals(VisibilityLevel.SNAPSHOT, state.getVisibilityLevel());
+ assertEquals(wp, state.getWritePointer()); // Make sure write ptr didn't move
+
+ rs = conn.createStatement().executeQuery("select /*+ NO_INDEX */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(1,rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(3,rs.getLong(1));
+ assertFalse(rs.next());
+
+ rs = conn.createStatement().executeQuery("select /*+ INDEX(DEMO IDX) */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(3,rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(1,rs.getLong(1));
+ assertFalse(rs.next());
+
+ conn.createStatement().execute("delete from " + fullTableName + "1 where id1 in (select fk1a from " + fullTableName + "1 join " + fullTableName + "2 on (fk2=id1))");
+ assertEquals(VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT, state.getVisibilityLevel());
+ assertNotEquals(wp, state.getWritePointer()); // Make sure write ptr moved
+
+ rs = conn.createStatement().executeQuery("select /*+ NO_INDEX */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(1,rs.getLong(1));
+ assertFalse(rs.next());
+
+ rs = conn.createStatement().executeQuery("select /*+ INDEX(DEMO IDX) */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(1,rs.getLong(1));
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName + "1 SELECT id1 + 3, id1, id1 FROM " + fullTableName + "1");
+ stmt.executeUpdate("upsert into " + fullTableName + "2 values (2, 4)");
+
+ conn.createStatement().execute("delete from " + fullTableName + "1 where id1 in (select fk1a from " + fullTableName + "1 join " + fullTableName + "2 on (fk2=id1))");
+ assertEquals(VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT, state.getVisibilityLevel());
+ assertNotEquals(wp, state.getWritePointer()); // Make sure write ptr moved
+
+ rs = conn.createStatement().executeQuery("select /*+ NO_INDEX */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(4,rs.getLong(1));
+ assertFalse(rs.next());
+
+ rs = conn.createStatement().executeQuery("select /*+ INDEX(DEMO IDX) */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(4,rs.getLong(1));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ rs = conn.createStatement().executeQuery("select /*+ NO_INDEX */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(1,rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(2,rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(3,rs.getLong(1));
+ assertFalse(rs.next());
+
+ rs = conn.createStatement().executeQuery("select /*+ INDEX(DEMO IDX) */ id1 from " + fullTableName + "1");
+ assertTrue(rs.next());
+ assertEquals(3,rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(2,rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(1,rs.getLong(1));
+ assertFalse(rs.next());
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/tx/TxPointInTimeQueryIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TxPointInTimeQueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/TxPointInTimeQueryIT.java
new file mode 100644
index 0000000..a3b3aa2
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TxPointInTimeQueryIT.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.tx;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import org.apache.phoenix.end2end.BaseClientManagedTimeIT;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TxPointInTimeQueryIT extends BaseClientManagedTimeIT {
+
+ protected long ts;
+
+ @Before
+ public void initTable() throws Exception {
+ ts = nextTimestamp();
+ }
+
+ @Test
+ public void testQueryWithSCN() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts));
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);) {
+ try {
+ conn.createStatement().execute(
+ "CREATE TABLE t (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR) TRANSACTIONAL=true");
+ fail();
+ } catch (SQLException e) {
+ assertEquals("Unexpected Exception",
+ SQLExceptionCode.CANNOT_START_TRANSACTION_WITH_SCN_SET
+ .getErrorCode(), e.getErrorCode());
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/cache/IndexMetaDataCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/cache/IndexMetaDataCache.java b/phoenix-core/src/main/java/org/apache/phoenix/cache/IndexMetaDataCache.java
index a1563db..359e08f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/IndexMetaDataCache.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/IndexMetaDataCache.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import java.util.Collections;
import java.util.List;
+import co.cask.tephra.Transaction;
+
import org.apache.phoenix.index.IndexMaintainer;
public interface IndexMetaDataCache extends Closeable {
@@ -36,6 +38,13 @@ public interface IndexMetaDataCache extends Closeable {
public List<IndexMaintainer> getIndexMaintainers() {
return Collections.emptyList();
}
+
+ @Override
+ public Transaction getTransaction() {
+ return null;
+ }
+
};
public List<IndexMaintainer> getIndexMaintainers();
+ public Transaction getTransaction();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java b/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
index 3cfdd71..f188ab2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
@@ -71,6 +71,7 @@ import org.apache.phoenix.util.SQLCloseables;
import org.apache.phoenix.util.ScanUtil;
import com.google.common.collect.ImmutableSet;
+import com.google.protobuf.HBaseZeroCopyByteString;
/**
*
@@ -144,7 +145,7 @@ public class ServerCacheClient {
}
- public ServerCache addServerCache(ScanRanges keyRanges, final ImmutableBytesWritable cachePtr, final ServerCacheFactory cacheFactory, final TableRef cacheUsingTableRef) throws SQLException {
+ public ServerCache addServerCache(ScanRanges keyRanges, final ImmutableBytesWritable cachePtr, final byte[] txState, final ServerCacheFactory cacheFactory, final TableRef cacheUsingTableRef) throws SQLException {
ConnectionQueryServices services = connection.getQueryServices();
MemoryChunk chunk = services.getMemoryManager().allocate(cachePtr.getLength());
List<Closeable> closeables = new ArrayList<Closeable>();
@@ -213,6 +214,7 @@ public class ServerCacheClient {
ServerCacheFactoryProtos.ServerCacheFactory.Builder svrCacheFactoryBuider = ServerCacheFactoryProtos.ServerCacheFactory.newBuilder();
svrCacheFactoryBuider.setClassName(cacheFactory.getClass().getName());
builder.setCacheFactory(svrCacheFactoryBuider.build());
+ builder.setTxState(HBaseZeroCopyByteString.wrap(txState));
instance.addServerCache(controller, builder.build(), rpcCallback);
if(controller.getFailedOn() != null) {
throw controller.getFailedOn();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCache.java b/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCache.java
index b968a9b..c7cd58f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCache.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCache.java
@@ -37,6 +37,6 @@ import org.apache.phoenix.memory.MemoryManager;
public interface TenantCache {
MemoryManager getMemoryManager();
Closeable getServerCache(ImmutableBytesPtr cacheId);
- Closeable addServerCache(ImmutableBytesPtr cacheId, ImmutableBytesWritable cachePtr, ServerCacheFactory cacheFactory) throws SQLException;
+ Closeable addServerCache(ImmutableBytesPtr cacheId, ImmutableBytesWritable cachePtr, byte[] txState, ServerCacheFactory cacheFactory) throws SQLException;
void removeServerCache(ImmutableBytesPtr cacheId) throws SQLException;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
index 9005fa8..6ef7a6f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
@@ -23,14 +23,17 @@ import java.util.Collections;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
-import com.google.common.cache.*;
import org.apache.phoenix.coprocessor.ServerCachingProtocol.ServerCacheFactory;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.memory.MemoryManager;
import org.apache.phoenix.memory.MemoryManager.MemoryChunk;
import org.apache.phoenix.util.Closeables;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+
/**
*
* Cache per tenant on server side. Tracks memory usage for each
@@ -80,11 +83,11 @@ public class TenantCacheImpl implements TenantCache {
}
@Override
- public Closeable addServerCache(ImmutableBytesPtr cacheId, ImmutableBytesWritable cachePtr, ServerCacheFactory cacheFactory) throws SQLException {
- MemoryChunk chunk = this.getMemoryManager().allocate(cachePtr.getLength());
+ public Closeable addServerCache(ImmutableBytesPtr cacheId, ImmutableBytesWritable cachePtr, byte[] txState, ServerCacheFactory cacheFactory) throws SQLException {
+ MemoryChunk chunk = this.getMemoryManager().allocate(cachePtr.getLength() + txState.length);
boolean success = false;
try {
- Closeable element = cacheFactory.newCache(cachePtr, chunk);
+ Closeable element = cacheFactory.newCache(cachePtr, txState, chunk);
getServerCaches().put(cacheId, element);
success = true;
return element;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
index f09b508..d5b74fb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
@@ -132,8 +132,8 @@ public class CreateTableCompiler {
// on our connection.
new DelegateConnectionQueryServices(connection.getQueryServices()) {
@Override
- public PMetaData addTable(PTable table) throws SQLException {
- return connection.addTable(table);
+ public PMetaData addTable(PTable table, long resolvedTime) throws SQLException {
+ return connection.addTable(table, resolvedTime);
}
},
connection, tableRef.getTimeStamp());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
index f0f693e..12aa7c5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
@@ -78,6 +78,7 @@ import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PLong;
+import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.ScanUtil;
@@ -323,16 +324,20 @@ public class DeleteCompiler {
// - read-only VIEW
// - transactional table with a connection having an SCN
// TODO: SchemaUtil.isReadOnly(PTable, connection)?
- if ( table.getType() == PTableType.VIEW && table.getViewType().isReadOnly() ) {
+ if (table.getType() == PTableType.VIEW && table.getViewType().isReadOnly()) {
throw new ReadOnlyTableException(schemaName,tableName);
}
+ else if (table.isTransactional() && connection.getSCN() != null) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_SPECIFY_SCN_FOR_TXN_TABLE).setSchemaName(schemaName)
+ .setTableName(tableName).build().buildException();
+ }
immutableIndex = getNonDisabledImmutableIndexes(tableRefToBe);
boolean mayHaveImmutableIndexes = !immutableIndex.isEmpty();
noQueryReqd = !hasLimit;
// Can't run on same server for transactional data, as we need the row keys for the data
// that is being upserted for conflict detection purposes.
- runOnServer = isAutoCommit && noQueryReqd;
+ runOnServer = isAutoCommit && noQueryReqd && !table.isTransactional();
HintNode hint = delete.getHint();
if (runOnServer && !delete.getHint().hasHint(Hint.USE_INDEX_OVER_DATA_TABLE)) {
hint = HintNode.create(hint, Hint.USE_DATA_OVER_INDEX_TABLE);
@@ -363,8 +368,8 @@ public class DeleteCompiler {
QueryCompiler compiler = new QueryCompiler(statement, select, resolverToBe, Collections.<PColumn>emptyList(), parallelIteratorFactory, new SequenceManager(statement));
dataPlanToBe = compiler.compile();
queryPlans = Lists.newArrayList(mayHaveImmutableIndexes
- ? optimizer.getApplicablePlans(statement, select, resolverToBe, Collections.<PColumn>emptyList(), parallelIteratorFactory)
- : optimizer.getBestPlan(statement, select, resolverToBe, Collections.<PColumn>emptyList(), parallelIteratorFactory));
+ ? optimizer.getApplicablePlans(dataPlanToBe, statement, select, resolverToBe, Collections.<PColumn>emptyList(), parallelIteratorFactory)
+ : optimizer.getBestPlan(dataPlanToBe, statement, select, resolverToBe, Collections.<PColumn>emptyList(), parallelIteratorFactory));
if (mayHaveImmutableIndexes) { // FIXME: this is ugly
// Lookup the table being deleted from in the cache, as it's possible that the
// optimizer updated the cache if it found indexes that were out of date.
@@ -538,11 +543,12 @@ public class DeleteCompiler {
ImmutableBytesWritable ptr = context.getTempPtr();
PTable table = tableRef.getTable();
table.getIndexMaintainers(ptr, context.getConnection());
+ byte[] txState = table.isTransactional() ? connection.getMutationState().encodeTransaction() : ByteUtil.EMPTY_BYTE_ARRAY;
ServerCache cache = null;
try {
if (ptr.getLength() > 0) {
IndexMetaDataCacheClient client = new IndexMetaDataCacheClient(connection, tableRef);
- cache = client.addIndexMetadataCache(context.getScanRanges(), ptr);
+ cache = client.addIndexMetadataCache(context.getScanRanges(), ptr, txState);
byte[] uuidValue = cache.getId();
context.getScan().setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
index 426556f..0828b94 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
@@ -80,6 +80,7 @@ import org.apache.phoenix.util.Closeables;
import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.LogUtil;
import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TransactionUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -226,7 +227,7 @@ public class FromCompiler {
PTable t = PTableImpl.makePTable(table, projectedColumns);
return new SingleTableColumnResolver(connection, new TableRef(tableRef.getTableAlias(), t, tableRef.getLowerBoundTimeStamp(), tableRef.hasDynamicCols()));
}
-
+
public static ColumnResolver getResolver(TableRef tableRef)
throws SQLException {
SingleTableColumnResolver visitor = new SingleTableColumnResolver(tableRef);
@@ -407,7 +408,7 @@ public class FromCompiler {
PTable theTable = null;
if (updateCacheImmediately || connection.getAutoCommit()) {
MetaDataMutationResult result = client.updateCache(schemaName, tableName);
- timeStamp = result.getMutationTime();
+ timeStamp = TransactionUtil.getResolvedTimestamp(connection, result);
theTable = result.getTable();
if (theTable == null) {
throw new TableNotFoundException(schemaName, tableName, timeStamp);
@@ -427,7 +428,7 @@ public class FromCompiler {
if (theTable == null) {
MetaDataMutationResult result = client.updateCache(schemaName, tableName);
if (result.wasUpdated()) {
- timeStamp = result.getMutationTime();
+ timeStamp = TransactionUtil.getResolvedTimestamp(connection, result);
theTable = result.getTable();
}
}
@@ -651,7 +652,7 @@ public class FromCompiler {
PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM,
null, null, columns, null, null, Collections.<PTable>emptyList(),
false, Collections.<PName>emptyList(), null, null, false, false, false, null,
- null, null, false);
+ null, null, false, false);
String alias = subselectNode.getAlias();
TableRef tableRef = new TableRef(alias, t, MetaDataProtocol.MIN_TABLE_TIMESTAMP, false);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
index e4ca5d9..b55e4aa 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
@@ -1302,7 +1302,7 @@ public class JoinCompiler {
left.getBucketNum(), merged,left.getParentSchemaName(), left.getParentTableName(), left.getIndexes(),
left.isImmutableRows(), Collections.<PName>emptyList(), null, null, PTable.DEFAULT_DISABLE_WAL,
left.isMultiTenant(), left.getStoreNulls(), left.getViewType(), left.getViewIndexId(), left.getIndexType(),
- left.rowKeyOrderOptimizable());
+ left.rowKeyOrderOptimizable(), left.isTransactional());
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
index c6f6bf2..506623b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
@@ -48,6 +48,7 @@ import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.util.ScanUtil;
+import org.apache.phoenix.util.TransactionUtil;
import com.google.common.collect.Lists;
@@ -179,7 +180,14 @@ public class PostDDLCompiler {
};
PhoenixStatement statement = new PhoenixStatement(connection);
StatementContext context = new StatementContext(statement, resolver, scan, new SequenceManager(statement));
- ScanUtil.setTimeRange(scan, timestamp);
+ long ts = timestamp;
+ // FIXME: DDL operations aren't transactional, so we're basing the timestamp on a server timestamp.
+ // Not sure what the fix should be. We don't need conflict detection nor filtering of invalid transactions
+ // in this case, so maybe this is ok.
+ if (tableRef.getTable().isTransactional()) {
+ ts = TransactionUtil.convertToNanoseconds(ts);
+ }
+ ScanUtil.setTimeRange(scan, ts);
if (emptyCF != null) {
scan.setAttribute(BaseScannerRegionObserver.EMPTY_CF, emptyCF);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
index 312de45..80c4b89 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
@@ -252,7 +252,8 @@ public class StatementContext {
public long getCurrentTime() throws SQLException {
long ts = this.getCurrentTable().getTimeStamp();
- if (ts != QueryConstants.UNSET_TIMESTAMP) {
+ // if the table is transactional then it is only resolved once per query, so we can't use the table timestamp
+ if (!this.getCurrentTable().getTable().isTransactional() && ts != QueryConstants.UNSET_TIMESTAMP) {
return ts;
}
if (currentTime != QueryConstants.UNSET_TIMESTAMP) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
index c6aa546..551b05c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
@@ -152,7 +152,7 @@ public class TupleProjectionCompiler {
table.getBucketNum(), projectedColumns, table.getParentSchemaName(),
table.getParentName(), table.getIndexes(), table.isImmutableRows(), Collections.<PName>emptyList(), null, null,
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(),
- table.getIndexType(), table.rowKeyOrderOptimizable());
+ table.getIndexType(), table.rowKeyOrderOptimizable(), table.isTransactional());
}
public static PTable createProjectedTable(TableRef tableRef, List<ColumnRef> sourceColumnRefs, boolean retainPKColumns) throws SQLException {
@@ -179,7 +179,7 @@ public class TupleProjectionCompiler {
retainPKColumns ? table.getBucketNum() : null, projectedColumns, null,
null, Collections.<PTable>emptyList(), table.isImmutableRows(), Collections.<PName>emptyList(), null, null,
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(),
- null, table.rowKeyOrderOptimizable());
+ null, table.rowKeyOrderOptimizable(), table.isTransactional());
}
// For extracting column references from single select statement
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
index afca97a..298303d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
@@ -82,7 +82,7 @@ public class UnionCompiler {
PTable tempTable = PTableImpl.makePTable(statement.getConnection().getTenantId(), UNION_SCHEMA_NAME, UNION_TABLE_NAME,
PTableType.SUBQUERY, null, HConstants.LATEST_TIMESTAMP, scn == null ? HConstants.LATEST_TIMESTAMP : scn, null, null,
projectedColumns, null, null, null,
- true, null, null, null, true, true, true, null, null, null, false);
+ true, null, null, null, true, true, true, null, null, null, false, false);
TableRef tableRef = new TableRef(null, tempTable, 0, false);
return tableRef;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
index 953eb2f..0f7f6f9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
@@ -182,7 +182,7 @@ public class UpsertCompiler {
if (isAutoCommit && rowCount % batchSize == 0) {
MutationState state = new MutationState(tableRef, mutation, 0, maxSize, connection);
connection.getMutationState().join(state);
- connection.commit();
+ connection.getMutationState().send();
mutation.clear();
}
}
@@ -292,9 +292,13 @@ public class UpsertCompiler {
// Cannot update:
// - read-only VIEW
// - transactional table with a connection having an SCN
- if ( table.getType() == PTableType.VIEW && table.getViewType().isReadOnly() ) {
+ if (table.getType() == PTableType.VIEW && table.getViewType().isReadOnly()) {
throw new ReadOnlyTableException(schemaName,tableName);
}
+ else if (table.isTransactional() && connection.getSCN() != null) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_SPECIFY_SCN_FOR_TXN_TABLE).setSchemaName(schemaName)
+ .setTableName(tableName).build().buildException();
+ }
boolean isSalted = table.getBucketNum() != null;
isTenantSpecific = table.isMultiTenant() && connection.getTenantId() != null;
isSharedViewIndex = table.getViewIndexId() != null;
@@ -463,7 +467,7 @@ public class UpsertCompiler {
// so we might be able to run it entirely on the server side.
// For a table with row timestamp column, we can't guarantee that the row key will reside in the
// region space managed by region servers. So we bail out on executing on server side.
- runOnServer = sameTable && isAutoCommit && !(table.isImmutableRows() && !table.getIndexes().isEmpty()) && table.getRowTimestampColPos() == -1;
+ runOnServer = sameTable && isAutoCommit && !table.isTransactional() && !(table.isImmutableRows() && !table.getIndexes().isEmpty()) && table.getRowTimestampColPos() == -1;
}
// If we may be able to run on the server, add a hint that favors using the data table
// if all else is equal.
@@ -678,12 +682,13 @@ public class UpsertCompiler {
ImmutableBytesWritable ptr = context.getTempPtr();
PTable table = tableRef.getTable();
table.getIndexMaintainers(ptr, context.getConnection());
+ byte[] txState = table.isTransactional() ? connection.getMutationState().encodeTransaction() : ByteUtil.EMPTY_BYTE_ARRAY;
ServerCache cache = null;
try {
if (ptr.getLength() > 0) {
IndexMetaDataCacheClient client = new IndexMetaDataCacheClient(connection, tableRef);
- cache = client.addIndexMetadataCache(context.getScanRanges(), ptr);
+ cache = client.addIndexMetadataCache(context.getScanRanges(), ptr, txState);
byte[] uuidValue = cache.getId();
scan.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index 33218ee..d720806 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -57,6 +57,8 @@ import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.ServerUtil;
+import co.cask.tephra.Transaction;
+
import com.google.common.collect.ImmutableList;
@@ -86,6 +88,7 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
public static final String EXPECTED_UPPER_REGION_KEY = "_ExpectedUpperRegionKey";
public static final String REVERSE_SCAN = "_ReverseScan";
public static final String ANALYZE_TABLE = "_ANALYZETABLE";
+ public static final String TX_STATE = "_TxState";
public static final String GUIDEPOST_WIDTH_BYTES = "_GUIDEPOST_WIDTH_BYTES";
public static final String GUIDEPOST_PER_REGION = "_GUIDEPOST_PER_REGION";
public static final String UPGRADE_DESC_ROW_KEY = "_UPGRADE_DESC_ROW_KEY";
@@ -228,7 +231,7 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
final byte[][] viewConstants, final TupleProjector projector,
final ImmutableBytesWritable ptr) {
return getWrappedScanner(c, s, null, null, offset, scan, dataColumns, tupleProjector,
- dataRegion, indexMaintainer, viewConstants, null, null, projector, ptr);
+ dataRegion, indexMaintainer, null, viewConstants, null, null, projector, ptr);
}
/**
@@ -236,13 +239,14 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
* re-throws as DoNotRetryIOException to prevent needless retrying hanging the query
* for 30 seconds. Unfortunately, until HBASE-7481 gets fixed, there's no way to do
* the same from a custom filter.
- * @param arrayFuncRefs
* @param arrayKVRefs
+ * @param arrayFuncRefs
* @param offset starting position in the rowkey.
* @param scan
* @param tupleProjector
* @param dataRegion
* @param indexMaintainer
+ * @param tx current transaction
* @param viewConstants
*/
protected RegionScanner getWrappedScanner(final ObserverContext<RegionCoprocessorEnvironment> c,
@@ -250,6 +254,7 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
final Expression[] arrayFuncRefs, final int offset, final Scan scan,
final ColumnReference[] dataColumns, final TupleProjector tupleProjector,
final Region dataRegion, final IndexMaintainer indexMaintainer,
+ Transaction tx,
final byte[][] viewConstants, final KeyValueSchema kvSchema,
final ValueBitSet kvSchemaBitSet, final TupleProjector projector,
final ImmutableBytesWritable ptr) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 7eb1dc6..0a313be 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -60,6 +60,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME_INDEX;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID_INDEX;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_BYTES;
@@ -240,6 +241,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
private static final KeyValue EMPTY_KEYVALUE_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES);
private static final KeyValue BASE_COLUMN_COUNT_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.BASE_COLUMN_COUNT_BYTES);
private static final KeyValue ROW_KEY_ORDER_OPTIMIZABLE_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, ROW_KEY_ORDER_OPTIMIZABLE_BYTES);
+ private static final KeyValue TRANSACTIONAL_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, TRANSACTIONAL_BYTES);
private static final List<KeyValue> TABLE_KV_COLUMNS = Arrays.<KeyValue>asList(
EMPTY_KEYVALUE_KV,
@@ -261,7 +263,8 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
INDEX_DISABLE_TIMESTAMP_KV,
STORE_NULLS_KV,
BASE_COLUMN_COUNT_KV,
- ROW_KEY_ORDER_OPTIMIZABLE_KV
+ ROW_KEY_ORDER_OPTIMIZABLE_KV,
+ TRANSACTIONAL_KV
);
static {
Collections.sort(TABLE_KV_COLUMNS, KeyValue.COMPARATOR);
@@ -285,6 +288,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
private static final int STORE_NULLS_INDEX = TABLE_KV_COLUMNS.indexOf(STORE_NULLS_KV);
private static final int BASE_COLUMN_COUNT_INDEX = TABLE_KV_COLUMNS.indexOf(BASE_COLUMN_COUNT_KV);
private static final int ROW_KEY_ORDER_OPTIMIZABLE_INDEX = TABLE_KV_COLUMNS.indexOf(ROW_KEY_ORDER_OPTIMIZABLE_KV);
+ private static final int TRANSACTIONAL_INDEX = TABLE_KV_COLUMNS.indexOf(TRANSACTIONAL_KV);
// KeyValues for Column
private static final KeyValue DECIMAL_DIGITS_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, DECIMAL_DIGITS_BYTES);
@@ -789,6 +793,8 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
boolean multiTenant = multiTenantKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(multiTenantKv.getValueArray(), multiTenantKv.getValueOffset(), multiTenantKv.getValueLength()));
Cell storeNullsKv = tableKeyValues[STORE_NULLS_INDEX];
boolean storeNulls = storeNullsKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(storeNullsKv.getValueArray(), storeNullsKv.getValueOffset(), storeNullsKv.getValueLength()));
+ Cell transactionalKv = tableKeyValues[TRANSACTIONAL_INDEX];
+ boolean transactional = transactionalKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(transactionalKv.getValueArray(), transactionalKv.getValueOffset(), transactionalKv.getValueLength()));
Cell viewTypeKv = tableKeyValues[VIEW_TYPE_INDEX];
ViewType viewType = viewTypeKv == null ? null : ViewType.fromSerializedValue(viewTypeKv.getValueArray()[viewTypeKv.getValueOffset()]);
Cell viewIndexIdKv = tableKeyValues[VIEW_INDEX_ID_INDEX];
@@ -844,7 +850,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
return PTableImpl.makePTable(tenantId, schemaName, tableName, tableType, indexState, timeStamp,
tableSeqNum, pkName, saltBucketNum, columns, tableType == INDEX ? schemaName : null,
tableType == INDEX ? dataTableName : null, indexes, isImmutableRows, physicalTables, defaultFamilyName, viewStatement,
- disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, stats, baseColumnCount, rowKeyOrderOptimizable);
+ disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, stats, baseColumnCount, rowKeyOrderOptimizable, transactional);
}
private PFunction getFunction(RegionScanner scanner, final boolean isReplace, long clientTimeStamp, List<Mutation> deleteMutationsForReplace)
@@ -2332,6 +2338,17 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
}
continue;
}
+ } else if (pkCount == COLUMN_NAME_INDEX &&
+ ! (Bytes.compareTo(schemaName, rowKeyMetaData[SCHEMA_NAME_INDEX]) == 0 &&
+ Bytes.compareTo(tableName, rowKeyMetaData[TABLE_NAME_INDEX]) == 0 ) ) {
+ // Invalidate any table with mutations
+ // TODO: this likely means we don't need the above logic that
+ // loops through the indexes if adding a PK column, since we'd
+ // always have header rows for those.
+ invalidateList.add(new ImmutableBytesPtr(SchemaUtil
+ .getTableKey(tenantId,
+ rowKeyMetaData[SCHEMA_NAME_INDEX],
+ rowKeyMetaData[TABLE_NAME_INDEX])));
}
}
tableMetaData.addAll(mutationsForAddingColumnsToViews);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index 7cc4123..ba06828 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -23,9 +23,9 @@ import java.util.List;
import org.apache.hadoop.hbase.util.ByteStringer;
import org.apache.phoenix.coprocessor.generated.MetaDataProtos;
-import org.apache.phoenix.coprocessor.generated.PFunctionProtos;
import org.apache.phoenix.coprocessor.generated.MetaDataProtos.MetaDataResponse;
import org.apache.phoenix.coprocessor.generated.MetaDataProtos.MetaDataService;
+import org.apache.phoenix.coprocessor.generated.PFunctionProtos;
import org.apache.phoenix.hbase.index.util.VersionUtil;
import org.apache.phoenix.parse.PFunction;
import org.apache.phoenix.schema.PColumn;
@@ -73,7 +73,9 @@ public abstract class MetaDataProtocol extends MetaDataService {
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0 = MIN_TABLE_TIMESTAMP + 7;
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0 = MIN_TABLE_TIMESTAMP + 8;
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0 = MIN_TABLE_TIMESTAMP + 9;
- public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0;
+ public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 = MIN_TABLE_TIMESTAMP + 10;
+ // MIN_SYSTEM_TABLE_TIMESTAMP needs to be set to the max of all the MIN_SYSTEM_TABLE_TIMESTAMP_* constants
+ public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0;
// TODO: pare this down to minimum, as we don't need duplicates for both table and column errors, nor should we need
// a different code for every type of error.
// ENTITY_ALREADY_EXISTS, ENTITY_NOT_FOUND, NEWER_ENTITY_FOUND, ENTITY_NOT_IN_REGION, CONCURRENT_MODIFICATION
[30/52] [abbrv] phoenix git commit: PHOENIX-2471 Move update meta
data call prior to getting index maintainer
Posted by ma...@apache.org.
PHOENIX-2471 Move update meta data call prior to getting index maintainer
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/49be33e7
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/49be33e7
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/49be33e7
Branch: refs/heads/calcite
Commit: 49be33e71b592e330f1304cfa20bbffc8bd18637
Parents: 15ab697
Author: James Taylor <jt...@salesforce.com>
Authored: Tue Dec 1 13:30:43 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Tue Dec 1 13:30:43 2015 -0800
----------------------------------------------------------------------
.../apache/phoenix/end2end/index/IndexIT.java | 85 ++++++++++++++++++++
.../apache/phoenix/execute/MutationState.java | 14 ++--
2 files changed, 92 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/49be33e7/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
index 003e564..72f2230 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
@@ -32,6 +32,7 @@ import org.apache.phoenix.parse.TableName;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.util.DateUtil;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
@@ -49,6 +50,7 @@ import com.google.common.collect.Maps;
public class IndexIT extends BaseHBaseManagedTimeIT {
private final boolean localIndex;
+ private final boolean transactional;
private final String tableDDLOptions;
private final String tableName;
private final String indexName;
@@ -57,6 +59,7 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
public IndexIT(boolean localIndex, boolean mutable, boolean transactional) {
this.localIndex = localIndex;
+ this.transactional = transactional;
StringBuilder optionBuilder = new StringBuilder();
if (!mutable)
optionBuilder.append(" IMMUTABLE_ROWS=true ");
@@ -200,6 +203,88 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
}
@Test
+ public void testCreateIndexAfterUpsertStarted() throws Exception {
+ if (transactional) { // FIXME: PHOENIX-2446
+ return;
+ }
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn1 = DriverManager.getConnection(getUrl(), props)) {
+ conn1.setAutoCommit(false);
+ String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
+ Statement stmt = conn1.createStatement();
+ stmt.execute(ddl);
+ BaseTest.populateTestTable(fullTableName);
+
+ ResultSet rs;
+
+ rs = conn1.createStatement().executeQuery("SELECT COUNT(*) FROM " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals(3,rs.getInt(1));
+
+ try (Connection conn2 = DriverManager.getConnection(getUrl(), props)) {
+
+ String upsert = "UPSERT INTO " + fullTableName
+ + " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+ PreparedStatement pstmt = conn2.prepareStatement(upsert);
+ pstmt.setString(1, "varchar4");
+ pstmt.setString(2, "char4");
+ pstmt.setInt(3, 4);
+ pstmt.setLong(4, 4L);
+ pstmt.setBigDecimal(5, new BigDecimal(4.0));
+ Date date = DateUtil.parseDate("2015-01-01 00:00:00");
+ pstmt.setDate(6, date);
+ pstmt.setString(7, "varchar_a");
+ pstmt.setString(8, "chara");
+ pstmt.setInt(9, 2);
+ pstmt.setLong(10, 2L);
+ pstmt.setBigDecimal(11, new BigDecimal(2.0));
+ pstmt.setDate(12, date);
+ pstmt.setString(13, "varchar_b");
+ pstmt.setString(14, "charb");
+ pstmt.setInt(15, 3);
+ pstmt.setLong(16, 3L);
+ pstmt.setBigDecimal(17, new BigDecimal(3.0));
+ pstmt.setDate(18, date);
+ pstmt.executeUpdate();
+
+ ddl = "CREATE " + (localIndex ? "LOCAL" : "") + " INDEX " + indexName + " ON " + fullTableName
+ + " (long_pk, varchar_pk)"
+ + " INCLUDE (long_col1, long_col2)";
+ stmt.execute(ddl);
+
+ /*
+ * Commit upsert after index created through different connection.
+ * This forces conn2 (which doesn't know about the index yet) to update the metadata
+ * at commit time, recognize the new index, and generate the correct metadata (or index
+ * rows for immutable indexes).
+ *
+ * FIXME: PHOENIX-2446. For transactional data, this is problematic because the index
+ * gets a timestamp *after* the commit timestamp of conn2 and thus won't be seen during
+ * the commit. Also, when the index is being built, the data hasn't yet been committed
+ * and thus won't be part of the initial index build.
+ */
+ conn2.commit();
+
+ rs = conn1.createStatement().executeQuery("SELECT COUNT(*) FROM " + fullIndexName);
+ assertTrue(rs.next());
+ assertEquals(4,rs.getInt(1));
+
+ String query = "SELECT /*+ NO_INDEX */ long_pk FROM " + fullTableName;
+ rs = conn1.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals(1L, rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(2L, rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(3L, rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(4L, rs.getLong(1));
+ assertFalse(rs.next());
+ }
+ }
+ }
+
+ @Test
public void testDeleteFromNonPKColumnIndex() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/49be33e7/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
index 3444054..39fb2f4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -84,6 +84,12 @@ import org.apache.phoenix.util.TransactionUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
import co.cask.tephra.Transaction;
import co.cask.tephra.Transaction.VisibilityLevel;
import co.cask.tephra.TransactionAware;
@@ -93,12 +99,6 @@ import co.cask.tephra.TransactionFailureException;
import co.cask.tephra.TransactionSystemClient;
import co.cask.tephra.hbase11.TransactionAwareHTable;
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
/**
*
* Tracks the uncommitted state
@@ -745,10 +745,10 @@ public class MutationState implements SQLCloseable {
if (table.isTransactional()) {
uncommittedPhysicalNames.add(table.getPhysicalName().getString());
}
- table.getIndexMaintainers(indexMetaDataPtr, connection);
boolean isDataTable = true;
// Validate as we go if transactional since we can undo if a problem occurs (which is unlikely)
long serverTimestamp = serverTimeStamps == null ? validate(tableRef, valuesMap) : serverTimeStamps[i++];
+ table.getIndexMaintainers(indexMetaDataPtr, connection);
Iterator<Pair<byte[],List<Mutation>>> mutationsIterator = addRowMutations(tableRef, valuesMap, serverTimestamp, false, sendAll);
while (mutationsIterator.hasNext()) {
Pair<byte[],List<Mutation>> pair = mutationsIterator.next();
[17/52] [abbrv] phoenix git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/phoenix
Posted by ma...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/phoenix
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3bfa38dd
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3bfa38dd
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3bfa38dd
Branch: refs/heads/calcite
Commit: 3bfa38dd992836d81d44f2c5676dd3a216f8d788
Parents: 7b2e1b1 cab39d7
Author: maryannxue <we...@intel.com>
Authored: Wed Nov 25 14:01:16 2015 -0500
Committer: maryannxue <we...@intel.com>
Committed: Wed Nov 25 14:01:16 2015 -0500
----------------------------------------------------------------------
bin/phoenix_utils.py | 15 +++++++++++++--
bin/queryserver.py | 4 +++-
2 files changed, 16 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[49/52] [abbrv] phoenix git commit: PHOENIX-1949 NPE while inserting
NULL in a VARCHAR array using UPSERT stmt (Ram)
Posted by ma...@apache.org.
PHOENIX-1949 NPE while inserting NULL in a VARCHAR array using UPSERT stmt
(Ram)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c2a1f744
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c2a1f744
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c2a1f744
Branch: refs/heads/calcite
Commit: c2a1f7445f3882464a58d07cb9fd15a914e13d36
Parents: bf06617
Author: ramkrishna <ra...@gmail.com>
Authored: Wed Dec 9 09:43:01 2015 +0530
Committer: ramkrishna <ra...@gmail.com>
Committed: Wed Dec 9 09:45:01 2015 +0530
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/ArrayIT.java | 27 ++++++++++++++++++++
.../org/apache/phoenix/schema/types/PChar.java | 6 ++---
.../apache/phoenix/schema/types/PVarchar.java | 6 ++---
3 files changed, 33 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2a1f744/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayIT.java
index 7400127..846507b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayIT.java
@@ -870,6 +870,33 @@ public class ArrayIT extends BaseClientManagedTimeIT {
}
@Test
+ public void testArrayWithVarCharArray() throws Exception {
+ Connection conn;
+ PreparedStatement stmt;
+ ResultSet rs;
+ long ts = nextTimestamp();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement().execute("CREATE TABLE t ( k VARCHAR PRIMARY KEY, a VARCHAR ARRAY[])");
+ conn.close();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30));
+ conn = DriverManager.getConnection(getUrl(), props);
+ stmt = conn.prepareStatement("UPSERT INTO t VALUES('a',ARRAY['a',null])");
+ int res = stmt.executeUpdate();
+ assertEquals(1, res);
+ conn.commit();
+ conn.close();
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 40));
+ conn = DriverManager.getConnection(getUrl(), props);
+ rs = conn.createStatement().executeQuery("SELECT ARRAY_ELEM(a,2) FROM t");
+ assertTrue(rs.next());
+ assertEquals(null, rs.getString(1));
+ conn.close();
+ }
+
+ @Test
public void testArraySelectSingleArrayElemWithCast() throws Exception {
Connection conn;
PreparedStatement stmt;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2a1f744/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PChar.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PChar.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PChar.java
index 6b26197..4170253 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PChar.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PChar.java
@@ -104,12 +104,12 @@ public class PChar extends PDataType<String> {
@Override
public Object toObject(byte[] bytes, int offset, int length, PDataType actualType, SortOrder sortOrder, Integer maxLength, Integer scale) {
- if (!actualType.isCoercibleTo(this)) { // TODO: have isCoercibleTo that takes bytes, offset?
- throwConstraintViolationException(actualType,this);
- }
if (length == 0) {
return null;
}
+ if (!actualType.isCoercibleTo(this)) {
+ throwConstraintViolationException(actualType, this);
+ }
length = StringUtil.getUnpaddedCharLength(bytes, offset, length, sortOrder);
if (sortOrder == SortOrder.DESC) {
bytes = SortOrder.invert(bytes, offset, length);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2a1f744/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PVarchar.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PVarchar.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PVarchar.java
index fa3dbad..2575115 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PVarchar.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PVarchar.java
@@ -59,12 +59,12 @@ public class PVarchar extends PDataType<String> {
@Override
public Object toObject(byte[] bytes, int offset, int length, PDataType actualType,
SortOrder sortOrder, Integer maxLength, Integer scale) {
- if (!actualType.isCoercibleTo(this)) {
- throwConstraintViolationException(actualType, this);
- }
if (length == 0) {
return null;
}
+ if (!actualType.isCoercibleTo(this)) {
+ throwConstraintViolationException(actualType, this);
+ }
if (sortOrder == SortOrder.DESC) {
bytes = SortOrder.invert(bytes, offset, length);
offset = 0;
[33/52] [abbrv] phoenix git commit: PHOENIX-2472 Modify pom to
include tephra dependencies in server jar
Posted by ma...@apache.org.
PHOENIX-2472 Modify pom to include tephra dependencies in server jar
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/6aa4cc61
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/6aa4cc61
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/6aa4cc61
Branch: refs/heads/calcite
Commit: 6aa4cc61a50976c2406e3c0a53b0ecd6d69496e4
Parents: b8faae5
Author: Thomas D'Silva <td...@salesforce.com>
Authored: Tue Dec 1 11:10:54 2015 -0800
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Wed Dec 2 12:11:27 2015 -0800
----------------------------------------------------------------------
bin/tephra | 0
phoenix-assembly/src/build/server-with-txns.xml | 48 --------------------
.../src/build/server-without-antlr.xml | 46 -------------------
phoenix-assembly/src/build/server.xml | 4 ++
4 files changed, 4 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/6aa4cc61/bin/tephra
----------------------------------------------------------------------
diff --git a/bin/tephra b/bin/tephra
old mode 100644
new mode 100755
http://git-wip-us.apache.org/repos/asf/phoenix/blob/6aa4cc61/phoenix-assembly/src/build/server-with-txns.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/server-with-txns.xml b/phoenix-assembly/src/build/server-with-txns.xml
deleted file mode 100644
index 2808a8c..0000000
--- a/phoenix-assembly/src/build/server-with-txns.xml
+++ /dev/null
@@ -1,48 +0,0 @@
-<?xml version='1.0'?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
- <!-- build the phoenix server side jar, that includes phoenix-hadoopX-compat, phoenix-hadoop-compat and antlr -->
- <id>server-with-txns</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <dependencySets>
- <dependencySet>
- <outputDirectory>/</outputDirectory>
- <unpack>true</unpack>
- <includes>
- <include>org.apache.phoenix:phoenix-core</include>
- <include>org.iq80.snappy:snappy</include>
- <include>org.antlr:antlr*</include>
- <include>co.cask.tephra:tephra*</include>
- <include>com.google.code.gson:gson</include>
- <include>com.google.inject.extensions:guice-assistedinject</include>
- <include>it.unimi.dsi:fastutil</include>
- <include>io.dropwizard.metrics:metrics-core</include>
- </includes>
- </dependencySet>
- </dependencySets>
-</assembly>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/6aa4cc61/phoenix-assembly/src/build/server-without-antlr.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/server-without-antlr.xml b/phoenix-assembly/src/build/server-without-antlr.xml
deleted file mode 100644
index 33cf0b7..0000000
--- a/phoenix-assembly/src/build/server-without-antlr.xml
+++ /dev/null
@@ -1,46 +0,0 @@
-<?xml version='1.0'?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
- <!-- build the phoenix server side jar, that includes phoenix-hadoopX-compat and phoenix-hadoop-compat. -->
- <id>server-without-antlr</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <dependencySets>
- <dependencySet>
- <outputDirectory>/</outputDirectory>
- <unpack>true</unpack>
- <includes>
- <include>org.apache.phoenix:phoenix-core</include>
- <include>org.iq80.snappy:snappy</include>
- <include>co.cask.tephra:tephra*</include>
- <include>com.google.code.gson:gson</include>
- <include>org.jruby.joni:joni</include>
- <include>org.jruby.jcodings:jcodings</include>
- </includes>
- </dependencySet>
- </dependencySets>
-</assembly>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/6aa4cc61/phoenix-assembly/src/build/server.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/server.xml b/phoenix-assembly/src/build/server.xml
index 141f97b..ef79486 100644
--- a/phoenix-assembly/src/build/server.xml
+++ b/phoenix-assembly/src/build/server.xml
@@ -42,6 +42,10 @@
<include>org.jruby.joni:joni</include>
<include>org.jruby.jcodings:jcodings</include>
<include>joda-time:joda-time</include>
+ <include>org.apache.twill:twill*</include>
+ <include>com.google.inject.extensions:guice-assistedinject</include>
+ <include>it.unimi.dsi:fastutil</include>
+ <include>io.dropwizard.metrics:metrics-core</include>
</includes>
</dependencySet>
</dependencySets>
[45/52] [abbrv] phoenix git commit: PHOENIX-2481 JSON bulkload tool
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java
index eae58ad..05fbab2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java
@@ -66,9 +66,9 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner;
-import org.apache.phoenix.mapreduce.CsvBulkLoadTool.TargetTableRef;
-import org.apache.phoenix.mapreduce.CsvBulkLoadTool.TargetTableRefFunctions;
-import org.apache.phoenix.mapreduce.bulkload.CsvTableRowkeyPair;
+import org.apache.phoenix.mapreduce.bulkload.TableRowkeyPair;
+import org.apache.phoenix.mapreduce.bulkload.TargetTableRef;
+import org.apache.phoenix.mapreduce.bulkload.TargetTableRefFunctions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -81,7 +81,7 @@ import com.google.common.collect.Sets;
* It has been adapted from {#link HFileOutputFormat2} but differs from the fact it creates
* HFiles for multiple tables.
*/
-public class MultiHfileOutputFormat extends FileOutputFormat<CsvTableRowkeyPair, Cell> {
+public class MultiHfileOutputFormat extends FileOutputFormat<TableRowkeyPair, Cell> {
private static final Logger LOG = LoggerFactory.getLogger(MultiHfileOutputFormat.class);
@@ -101,7 +101,7 @@ public class MultiHfileOutputFormat extends FileOutputFormat<CsvTableRowkeyPair,
private static final String AT_DELIMITER = "@";
@Override
- public RecordWriter<CsvTableRowkeyPair, Cell> getRecordWriter(TaskAttemptContext context)
+ public RecordWriter<TableRowkeyPair, Cell> getRecordWriter(TaskAttemptContext context)
throws IOException, InterruptedException {
return createRecordWriter(context);
}
@@ -112,7 +112,7 @@ public class MultiHfileOutputFormat extends FileOutputFormat<CsvTableRowkeyPair,
* @return
* @throws IOException
*/
- static <V extends Cell> RecordWriter<CsvTableRowkeyPair, V> createRecordWriter(final TaskAttemptContext context)
+ static <V extends Cell> RecordWriter<TableRowkeyPair, V> createRecordWriter(final TaskAttemptContext context)
throws IOException {
// Get the path of the temporary output file
final Path outputPath = FileOutputFormat.getOutputPath(context);
@@ -130,7 +130,7 @@ public class MultiHfileOutputFormat extends FileOutputFormat<CsvTableRowkeyPair,
final boolean compactionExclude = conf.getBoolean(
"hbase.mapreduce.hfileoutputformat.compaction.exclude", false);
- return new RecordWriter<CsvTableRowkeyPair, V>() {
+ return new RecordWriter<TableRowkeyPair, V>() {
// Map of families to writers and how much has been output on the writer.
private final Map<byte [], WriterLength> writers =
new TreeMap<byte [], WriterLength>(Bytes.BYTES_COMPARATOR);
@@ -139,7 +139,7 @@ public class MultiHfileOutputFormat extends FileOutputFormat<CsvTableRowkeyPair,
private boolean rollRequested = false;
@Override
- public void write(CsvTableRowkeyPair row, V cell)
+ public void write(TableRowkeyPair row, V cell)
throws IOException {
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
// null input == user explicitly wants to flush
@@ -450,7 +450,7 @@ public class MultiHfileOutputFormat extends FileOutputFormat<CsvTableRowkeyPair,
* Configure <code>job</code> with a TotalOrderPartitioner, partitioning against
* <code>splitPoints</code>. Cleans up the partitions file after job exists.
*/
- static void configurePartitioner(Job job, Set<CsvTableRowkeyPair> tablesStartKeys)
+ static void configurePartitioner(Job job, Set<TableRowkeyPair> tablesStartKeys)
throws IOException {
Configuration conf = job.getConfiguration();
@@ -467,7 +467,7 @@ public class MultiHfileOutputFormat extends FileOutputFormat<CsvTableRowkeyPair,
}
private static void writePartitions(Configuration conf, Path partitionsPath,
- Set<CsvTableRowkeyPair> tablesStartKeys) throws IOException {
+ Set<TableRowkeyPair> tablesStartKeys) throws IOException {
LOG.info("Writing partition information to " + partitionsPath);
if (tablesStartKeys.isEmpty()) {
@@ -478,9 +478,9 @@ public class MultiHfileOutputFormat extends FileOutputFormat<CsvTableRowkeyPair,
// have keys < the first region (which has an empty start key)
// so we need to remove it. Otherwise we would end up with an
// empty reducer with index 0
- TreeSet<CsvTableRowkeyPair> sorted = new TreeSet<CsvTableRowkeyPair>(tablesStartKeys);
+ TreeSet<TableRowkeyPair> sorted = new TreeSet<TableRowkeyPair>(tablesStartKeys);
- CsvTableRowkeyPair first = sorted.first();
+ TableRowkeyPair first = sorted.first();
if (!first.getRowkey().equals(HConstants.EMPTY_BYTE_ARRAY)) {
throw new IllegalArgumentException(
"First region of table should have empty start key. Instead has: "
@@ -491,11 +491,11 @@ public class MultiHfileOutputFormat extends FileOutputFormat<CsvTableRowkeyPair,
// Write the actual file
FileSystem fs = partitionsPath.getFileSystem(conf);
SequenceFile.Writer writer = SequenceFile.createWriter(
- fs, conf, partitionsPath, CsvTableRowkeyPair.class,
+ fs, conf, partitionsPath, TableRowkeyPair.class,
NullWritable.class);
try {
- for (CsvTableRowkeyPair startKey : sorted) {
+ for (TableRowkeyPair startKey : sorted) {
writer.append(startKey, NullWritable.get());
}
} finally {
@@ -658,11 +658,11 @@ public class MultiHfileOutputFormat extends FileOutputFormat<CsvTableRowkeyPair,
KeyValueSerialization.class.getName());
// tableStartKeys for all tables.
- Set<CsvTableRowkeyPair> tablesStartKeys = Sets.newTreeSet();
+ Set<TableRowkeyPair> tablesStartKeys = Sets.newTreeSet();
for(TargetTableRef table : tablesToBeLoaded) {
final String tableName = table.getPhysicalName();
try(HTable htable = new HTable(conf,tableName);){
- Set<CsvTableRowkeyPair> startKeys = getRegionStartKeys(tableName , htable.getRegionLocator());
+ Set<TableRowkeyPair> startKeys = getRegionStartKeys(tableName , htable.getRegionLocator());
tablesStartKeys.addAll(startKeys);
String compressionConfig = configureCompression(htable.getTableDescriptor());
String bloomTypeConfig = configureBloomType(htable.getTableDescriptor());
@@ -704,12 +704,12 @@ public class MultiHfileOutputFormat extends FileOutputFormat<CsvTableRowkeyPair,
* Return the start keys of all of the regions in this table,
* as a list of ImmutableBytesWritable.
*/
- private static Set<CsvTableRowkeyPair> getRegionStartKeys(String tableName , RegionLocator table) throws IOException {
+ private static Set<TableRowkeyPair> getRegionStartKeys(String tableName , RegionLocator table) throws IOException {
byte[][] byteKeys = table.getStartKeys();
- Set<CsvTableRowkeyPair> ret = new TreeSet<CsvTableRowkeyPair>();
+ Set<TableRowkeyPair> ret = new TreeSet<TableRowkeyPair>();
for (byte[] byteKey : byteKeys) {
// phoenix-2216: start : passing the table name and startkey
- ret.add(new CsvTableRowkeyPair(tableName, new ImmutableBytesWritable(byteKey)));
+ ret.add(new TableRowkeyPair(tableName, new ImmutableBytesWritable(byteKey)));
}
return ret;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/CsvTableRowkeyPair.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/CsvTableRowkeyPair.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/CsvTableRowkeyPair.java
deleted file mode 100644
index 3ae74b6..0000000
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/CsvTableRowkeyPair.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.mapreduce.bulkload;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.io.WritableUtils;
-
-import com.google.common.base.Preconditions;
-
-
-/**
- * A WritableComparable to hold the table name and the rowkey.
- *
- */
-public class CsvTableRowkeyPair implements WritableComparable<CsvTableRowkeyPair> {
-
- /* The qualified table name */
- private String tableName;
-
- /* The rowkey for the record */
- private ImmutableBytesWritable rowkey;
-
- /**
- * Default constructor
- */
- public CsvTableRowkeyPair() {
- super();
- }
-
- /**
- * @param tableName
- * @param rowkey
- */
- public CsvTableRowkeyPair(String tableName, ImmutableBytesWritable rowkey) {
- super();
- Preconditions.checkNotNull(tableName);
- Preconditions.checkNotNull(rowkey);
- this.tableName = tableName;
- this.rowkey = rowkey;
- }
-
- public String getTableName() {
- return tableName;
- }
-
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
- public ImmutableBytesWritable getRowkey() {
- return rowkey;
- }
-
- public void setRowkey(ImmutableBytesWritable rowkey) {
- this.rowkey = rowkey;
- }
-
- @Override
- public void readFields(DataInput input) throws IOException {
- tableName = WritableUtils.readString(input);
- rowkey = new ImmutableBytesWritable();
- rowkey.readFields(input);
- }
-
- @Override
- public void write(DataOutput output) throws IOException {
- WritableUtils.writeString(output,tableName);
- rowkey.write(output);
- }
-
- @Override
- public int compareTo(CsvTableRowkeyPair other) {
- String otherTableName = other.getTableName();
- if(this.tableName.equals(otherTableName)) {
- return this.rowkey.compareTo(other.getRowkey());
- } else {
- return this.tableName.compareTo(otherTableName);
- }
- }
-
- /** Comparator optimized for <code>CsvTableRowkeyPair</code>. */
- public static class Comparator extends WritableComparator {
- private BytesWritable.Comparator comparator = new BytesWritable.Comparator();
-
- public Comparator() {
- super(CsvTableRowkeyPair.class);
- }
-
- @Override
- public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- try {
- int vintL1 = WritableUtils.decodeVIntSize(b1[s1]);
- int vintL2 = WritableUtils.decodeVIntSize(b2[s2]);
- int strL1 = readVInt(b1, s1);
- int strL2 = readVInt(b2, s2);
- int cmp = compareBytes(b1, s1 + vintL1, strL1, b2, s2 + vintL2, strL2);
- if (cmp != 0) {
- return cmp;
- }
- int vintL3 = WritableUtils.decodeVIntSize(b1[s1 + vintL1 + strL1]);
- int vintL4 = WritableUtils.decodeVIntSize(b2[s2 + vintL2 + strL2]);
- int strL3 = readVInt(b1, s1 + vintL1 + strL1);
- int strL4 = readVInt(b2, s2 + vintL2 + strL2);
- return comparator.compare(b1, s1 + vintL1 + strL1 + vintL3, strL3, b2, s2
- + vintL2 + strL2 + vintL4, strL4);
-
- } catch(Exception ex) {
- throw new IllegalArgumentException(ex);
- }
- }
- }
-
- static {
- WritableComparator.define(CsvTableRowkeyPair.class, new Comparator());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
new file mode 100644
index 0000000..412226f
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TableRowkeyPair.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce.bulkload;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.io.WritableUtils;
+
+import com.google.common.base.Preconditions;
+
+
+/**
+ * A WritableComparable to hold the table name and the rowkey.
+ */
+public class TableRowkeyPair implements WritableComparable<TableRowkeyPair> {
+
+ /* The qualified table name */
+ private String tableName;
+
+ /* The rowkey for the record */
+ private ImmutableBytesWritable rowkey;
+
+ /**
+ * Default constructor
+ */
+ public TableRowkeyPair() {
+ super();
+ }
+
+ public TableRowkeyPair(String tableName, ImmutableBytesWritable rowkey) {
+ super();
+ Preconditions.checkNotNull(tableName);
+ Preconditions.checkNotNull(rowkey);
+ this.tableName = tableName;
+ this.rowkey = rowkey;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public ImmutableBytesWritable getRowkey() {
+ return rowkey;
+ }
+
+ public void setRowkey(ImmutableBytesWritable rowkey) {
+ this.rowkey = rowkey;
+ }
+
+ @Override
+ public void readFields(DataInput input) throws IOException {
+ tableName = WritableUtils.readString(input);
+ rowkey = new ImmutableBytesWritable();
+ rowkey.readFields(input);
+ }
+
+ @Override
+ public void write(DataOutput output) throws IOException {
+ WritableUtils.writeString(output,tableName);
+ rowkey.write(output);
+ }
+
+ @Override
+ public int compareTo(TableRowkeyPair other) {
+ String otherTableName = other.getTableName();
+ if(this.tableName.equals(otherTableName)) {
+ return this.rowkey.compareTo(other.getRowkey());
+ } else {
+ return this.tableName.compareTo(otherTableName);
+ }
+ }
+
+ /** Comparator optimized for <code>TableRowkeyPair</code>. */
+ public static class Comparator extends WritableComparator {
+ private BytesWritable.Comparator comparator = new BytesWritable.Comparator();
+
+ public Comparator() {
+ super(TableRowkeyPair.class);
+ }
+
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ try {
+ int vintL1 = WritableUtils.decodeVIntSize(b1[s1]);
+ int vintL2 = WritableUtils.decodeVIntSize(b2[s2]);
+ int strL1 = readVInt(b1, s1);
+ int strL2 = readVInt(b2, s2);
+ int cmp = compareBytes(b1, s1 + vintL1, strL1, b2, s2 + vintL2, strL2);
+ if (cmp != 0) {
+ return cmp;
+ }
+ int vintL3 = WritableUtils.decodeVIntSize(b1[s1 + vintL1 + strL1]);
+ int vintL4 = WritableUtils.decodeVIntSize(b2[s2 + vintL2 + strL2]);
+ int strL3 = readVInt(b1, s1 + vintL1 + strL1);
+ int strL4 = readVInt(b2, s2 + vintL2 + strL2);
+ return comparator.compare(b1, s1 + vintL1 + strL1 + vintL3, strL3, b2, s2
+ + vintL2 + strL2 + vintL4, strL4);
+
+ } catch(Exception ex) {
+ throw new IllegalArgumentException(ex);
+ }
+ }
+ }
+
+ static {
+ WritableComparator.define(TableRowkeyPair.class, new Comparator());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TargetTableRef.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TargetTableRef.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TargetTableRef.java
new file mode 100644
index 0000000..1a846f9
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TargetTableRef.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce.bulkload;
+
+import java.util.Map;
+
+import org.codehaus.jackson.annotate.JsonCreator;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.google.common.collect.Maps;
+
+/**
+ * Represents the logical and physical name of a single table to which data is to be loaded.
+ *
+ * This class exists to allow for the difference between HBase physical table names and
+ * Phoenix logical table names.
+ */
+public class TargetTableRef {
+
+ @JsonProperty
+ private final String logicalName;
+
+ @JsonProperty
+ private final String physicalName;
+
+ @JsonProperty
+ private Map<String,String> configuration = Maps.newHashMap();
+
+ public TargetTableRef(String name) {
+ this(name, name);
+ }
+
+ @JsonCreator
+ private TargetTableRef(@JsonProperty("logicalName") String logicalName,
+ @JsonProperty("physicalName") String physicalName) {
+ this.logicalName = logicalName;
+ this.physicalName = physicalName;
+ }
+
+ public String getLogicalName() {
+ return logicalName;
+ }
+
+ public String getPhysicalName() {
+ return physicalName;
+ }
+
+ public Map<String, String> getConfiguration() {
+ return configuration;
+ }
+
+ public void setConfiguration(Map<String, String> configuration) {
+ this.configuration = configuration;
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TargetTableRefFunctions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TargetTableRefFunctions.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TargetTableRefFunctions.java
new file mode 100644
index 0000000..d786842
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/bulkload/TargetTableRefFunctions.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce.bulkload;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.codehaus.jackson.map.ObjectMapper;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+
+/**
+ * Utility functions to get/put json.
+ *
+ */
+public class TargetTableRefFunctions {
+
+ public static Function<TargetTableRef,String> TO_JSON = new Function<TargetTableRef,String>() {
+
+ @Override
+ public String apply(TargetTableRef input) {
+ try {
+ ObjectMapper mapper = new ObjectMapper();
+ return mapper.writeValueAsString(input);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+ };
+
+ public static Function<String,TargetTableRef> FROM_JSON = new Function<String,TargetTableRef>() {
+
+ @Override
+ public TargetTableRef apply(String json) {
+ try {
+ ObjectMapper mapper = new ObjectMapper();
+ return mapper.readValue(json, TargetTableRef.class);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+ };
+
+ public static Function<List<TargetTableRef>,String> NAMES_TO_JSON = new Function<List<TargetTableRef>,String>() {
+
+ @Override
+ public String apply(List<TargetTableRef> input) {
+ try {
+ List<String> tableNames = Lists.newArrayListWithCapacity(input.size());
+ for(TargetTableRef table : input) {
+ tableNames.add(table.getPhysicalName());
+ }
+ ObjectMapper mapper = new ObjectMapper();
+ return mapper.writeValueAsString(tableNames);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+ };
+
+ public static Function<String,List<String>> NAMES_FROM_JSON = new Function<String,List<String>>() {
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<String> apply(String json) {
+ try {
+ ObjectMapper mapper = new ObjectMapper();
+ return mapper.readValue(json, ArrayList.class);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+ };
+ }
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
index 8bf786b..9e29fba 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
@@ -23,13 +23,6 @@ import java.sql.SQLException;
import java.util.List;
import java.util.Map;
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Splitter;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -41,13 +34,17 @@ import org.apache.hadoop.mapreduce.lib.db.DBInputFormat.NullDBWritable;
import org.apache.hadoop.mapreduce.lib.db.DBWritable;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.phoenix.jdbc.PhoenixConnection;
-import org.apache.phoenix.mapreduce.CsvToKeyValueMapper.DefaultImportPreUpsertKeyValueProcessor;
+import org.apache.phoenix.mapreduce.FormatToKeyValueMapper;
import org.apache.phoenix.mapreduce.ImportPreUpsertKeyValueProcessor;
import org.apache.phoenix.mapreduce.PhoenixInputFormat;
import org.apache.phoenix.util.ColumnInfo;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.QueryUtil;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
import static org.apache.commons.lang.StringUtils.isNotEmpty;
/**
@@ -419,7 +416,7 @@ public final class PhoenixConfigurationUtil {
Class<? extends ImportPreUpsertKeyValueProcessor> processorClass = null;
try {
processorClass = conf.getClass(
- UPSERT_HOOK_CLASS_CONFKEY, DefaultImportPreUpsertKeyValueProcessor.class,
+ UPSERT_HOOK_CLASS_CONFKEY, FormatToKeyValueMapper.DefaultImportPreUpsertKeyValueProcessor.class,
ImportPreUpsertKeyValueProcessor.class);
} catch (Exception e) {
throw new IllegalStateException("Couldn't load upsert hook class", e);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/util/CSVCommonsLoader.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/CSVCommonsLoader.java b/phoenix-core/src/main/java/org/apache/phoenix/util/CSVCommonsLoader.java
index b8b284a..cdd9d7b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/CSVCommonsLoader.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/CSVCommonsLoader.java
@@ -17,36 +17,25 @@
*/
package org.apache.phoenix.util;
-import com.google.common.base.Charsets;
-import com.google.common.base.Joiner;
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.io.File;
+import java.io.Reader;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
-import org.apache.phoenix.exception.SQLExceptionCode;
-import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.jdbc.PhoenixConnection;
-import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.util.csv.CsvUpsertExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.Reader;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
+import com.google.common.base.Charsets;
+import com.google.common.base.Joiner;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
/***
* Upserts CSV data using Phoenix JDBC connection
@@ -222,7 +211,7 @@ public class CSVCommonsLoader {
long start = System.currentTimeMillis();
CsvUpsertListener upsertListener = new CsvUpsertListener(conn,
conn.getMutateBatchSize(), isStrict);
- CsvUpsertExecutor csvUpsertExecutor = CsvUpsertExecutor.create(conn, tableName,
+ CsvUpsertExecutor csvUpsertExecutor = new CsvUpsertExecutor(conn, tableName,
columnInfoList, upsertListener, arrayElementSeparator);
csvUpsertExecutor.execute(csvParser);
@@ -267,131 +256,10 @@ public class CSVCommonsLoader {
default:
throw new IllegalStateException("parser has unknown column source.");
}
- return generateColumnInfo(conn, tableName, columns, isStrict);
- }
-
- /**
- * Get list of ColumnInfos that contain Column Name and its associated
- * PDataType for an import. The supplied list of columns can be null -- if it is non-null,
- * it represents a user-supplied list of columns to be imported.
- *
- * @param conn Phoenix connection from which metadata will be read
- * @param tableName Phoenix table name whose columns are to be checked. Can include a schema
- * name
- * @param columns user-supplied list of import columns, can be null
- * @param strict if true, an exception will be thrown if unknown columns are supplied
- */
- public static List<ColumnInfo> generateColumnInfo(Connection conn,
- String tableName, List<String> columns, boolean strict)
- throws SQLException {
- Map<String, Integer> columnNameToTypeMap = Maps.newLinkedHashMap();
- Set<String> ambiguousColumnNames = new HashSet<String>();
- Map<String, Integer> fullColumnNameToTypeMap = Maps.newLinkedHashMap();
- DatabaseMetaData dbmd = conn.getMetaData();
- int unfoundColumnCount = 0;
- // TODO: escape wildcard characters here because we don't want that
- // behavior here
- String escapedTableName = StringUtil.escapeLike(tableName);
- String[] schemaAndTable = escapedTableName.split("\\.");
- ResultSet rs = null;
- try {
- rs = dbmd.getColumns(null, (schemaAndTable.length == 1 ? ""
- : schemaAndTable[0]),
- (schemaAndTable.length == 1 ? escapedTableName
- : schemaAndTable[1]), null);
- while (rs.next()) {
- String colName = rs.getString(QueryUtil.COLUMN_NAME_POSITION);
- String colFam = rs.getString(QueryUtil.COLUMN_FAMILY_POSITION);
-
- // use family qualifier, if available, otherwise, use column name
- String fullColumn = (colFam==null?colName:String.format("%s.%s",colFam,colName));
- String sqlTypeName = rs.getString(QueryUtil.DATA_TYPE_NAME_POSITION);
-
- // allow for both bare and family qualified names.
- if (columnNameToTypeMap.keySet().contains(colName)) {
- ambiguousColumnNames.add(colName);
- }
- columnNameToTypeMap.put(
- colName,
- PDataType.fromSqlTypeName(sqlTypeName).getSqlType());
- fullColumnNameToTypeMap.put(
- fullColumn,
- PDataType.fromSqlTypeName(sqlTypeName).getSqlType());
- }
- if (columnNameToTypeMap.isEmpty()) {
- throw new IllegalArgumentException("Table " + tableName + " not found");
- }
- } finally {
- if (rs != null) {
- rs.close();
- }
- }
- List<ColumnInfo> columnInfoList = Lists.newArrayList();
- Set<String> unresolvedColumnNames = new TreeSet<String>();
- if (columns == null) {
- // use family qualified names by default, if no columns are specified.
- for (Map.Entry<String, Integer> entry : fullColumnNameToTypeMap
- .entrySet()) {
- columnInfoList.add(new ColumnInfo(entry.getKey(), entry.getValue()));
- }
- } else {
- // Leave "null" as indication to skip b/c it doesn't exist
- for (int i = 0; i < columns.size(); i++) {
- String columnName = columns.get(i).trim();
- Integer sqlType = null;
- if (fullColumnNameToTypeMap.containsKey(columnName)) {
- sqlType = fullColumnNameToTypeMap.get(columnName);
- } else if (columnNameToTypeMap.containsKey(columnName)) {
- if (ambiguousColumnNames.contains(columnName)) {
- unresolvedColumnNames.add(columnName);
- }
- // fall back to bare column name.
- sqlType = columnNameToTypeMap.get(columnName);
- }
- if (unresolvedColumnNames.size()>0) {
- StringBuilder exceptionMessage = new StringBuilder();
- boolean first = true;
- exceptionMessage.append("Unable to resolve these column names to a single column family:\n");
- for (String col : unresolvedColumnNames) {
- if (first) first = false;
- else exceptionMessage.append(",");
- exceptionMessage.append(col);
- }
- exceptionMessage.append("\nAvailable columns with column families:\n");
- first = true;
- for (String col : fullColumnNameToTypeMap.keySet()) {
- if (first) first = false;
- else exceptionMessage.append(",");
- exceptionMessage.append(col);
- }
- throw new SQLException(exceptionMessage.toString());
- }
-
- if (sqlType == null) {
- if (strict) {
- throw new SQLExceptionInfo.Builder(
- SQLExceptionCode.COLUMN_NOT_FOUND)
- .setColumnName(columnName)
- .setTableName(tableName).build()
- .buildException();
- }
- unfoundColumnCount++;
- } else {
- columnInfoList.add(new ColumnInfo(columnName, sqlType));
- }
- }
- if (unfoundColumnCount == columns.size()) {
- throw new SQLExceptionInfo.Builder(
- SQLExceptionCode.COLUMN_NOT_FOUND)
- .setColumnName(
- Arrays.toString(columns.toArray(new String[0])))
- .setTableName(tableName).build().buildException();
- }
- }
- return columnInfoList;
+ return SchemaUtil.generateColumnInfo(conn, tableName, columns, isStrict);
}
- static class CsvUpsertListener implements CsvUpsertExecutor.UpsertListener {
+ static class CsvUpsertListener implements UpsertExecutor.UpsertListener<CSVRecord> {
private final PhoenixConnection conn;
private final int upsertBatchSize;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
index 1693600..cb8aced 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
@@ -17,21 +17,20 @@
*/
package org.apache.phoenix.util;
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Strings.isNullOrEmpty;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_NAME_BYTES;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES;
-
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-
+import java.util.Set;
+import java.util.TreeSet;
import javax.annotation.Nullable;
import org.apache.hadoop.hbase.KeyValue;
@@ -64,6 +63,15 @@ import org.apache.phoenix.schema.types.PVarbinary;
import org.apache.phoenix.schema.types.PVarchar;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_NAME_BYTES;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES;
/**
*
@@ -757,4 +765,125 @@ public class SchemaUtil {
public static byte getSeparatorByte(boolean rowKeyOrderOptimizable, boolean isNullValue, Expression e) {
return getSeparatorByte(rowKeyOrderOptimizable, isNullValue, e.getSortOrder());
}
+
+ /**
+ * Get list of ColumnInfos that contain Column Name and its associated
+ * PDataType for an import. The supplied list of columns can be null -- if it is non-null,
+ * it represents a user-supplied list of columns to be imported.
+ *
+ * @param conn Phoenix connection from which metadata will be read
+ * @param tableName Phoenix table name whose columns are to be checked. Can include a schema
+ * name
+ * @param columns user-supplied list of import columns, can be null
+ * @param strict if true, an exception will be thrown if unknown columns are supplied
+ */
+ public static List<ColumnInfo> generateColumnInfo(Connection conn,
+ String tableName, List<String> columns, boolean strict)
+ throws SQLException {
+ Map<String, Integer> columnNameToTypeMap = Maps.newLinkedHashMap();
+ Set<String> ambiguousColumnNames = new HashSet<String>();
+ Map<String, Integer> fullColumnNameToTypeMap = Maps.newLinkedHashMap();
+ DatabaseMetaData dbmd = conn.getMetaData();
+ int unfoundColumnCount = 0;
+ // TODO: escape wildcard characters here because we don't want that
+ // behavior here
+ String escapedTableName = StringUtil.escapeLike(tableName);
+ String[] schemaAndTable = escapedTableName.split("\\.");
+ ResultSet rs = null;
+ try {
+ rs = dbmd.getColumns(null, (schemaAndTable.length == 1 ? ""
+ : schemaAndTable[0]),
+ (schemaAndTable.length == 1 ? escapedTableName
+ : schemaAndTable[1]), null);
+ while (rs.next()) {
+ String colName = rs.getString(QueryUtil.COLUMN_NAME_POSITION);
+ String colFam = rs.getString(QueryUtil.COLUMN_FAMILY_POSITION);
+
+ // use family qualifier, if available, otherwise, use column name
+ String fullColumn = (colFam==null?colName:String.format("%s.%s",colFam,colName));
+ String sqlTypeName = rs.getString(QueryUtil.DATA_TYPE_NAME_POSITION);
+
+ // allow for both bare and family qualified names.
+ if (columnNameToTypeMap.keySet().contains(colName)) {
+ ambiguousColumnNames.add(colName);
+ }
+ columnNameToTypeMap.put(
+ colName,
+ PDataType.fromSqlTypeName(sqlTypeName).getSqlType());
+ fullColumnNameToTypeMap.put(
+ fullColumn,
+ PDataType.fromSqlTypeName(sqlTypeName).getSqlType());
+ }
+ if (columnNameToTypeMap.isEmpty()) {
+ throw new IllegalArgumentException("Table " + tableName + " not found");
+ }
+ } finally {
+ if (rs != null) {
+ rs.close();
+ }
+ }
+ List<ColumnInfo> columnInfoList = Lists.newArrayList();
+ Set<String> unresolvedColumnNames = new TreeSet<String>();
+ if (columns == null) {
+ // use family qualified names by default, if no columns are specified.
+ for (Map.Entry<String, Integer> entry : fullColumnNameToTypeMap
+ .entrySet()) {
+ columnInfoList.add(new ColumnInfo(entry.getKey(), entry.getValue()));
+ }
+ } else {
+ // Leave "null" as indication to skip b/c it doesn't exist
+ for (int i = 0; i < columns.size(); i++) {
+ String columnName = columns.get(i).trim();
+ Integer sqlType = null;
+ if (fullColumnNameToTypeMap.containsKey(columnName)) {
+ sqlType = fullColumnNameToTypeMap.get(columnName);
+ } else if (columnNameToTypeMap.containsKey(columnName)) {
+ if (ambiguousColumnNames.contains(columnName)) {
+ unresolvedColumnNames.add(columnName);
+ }
+ // fall back to bare column name.
+ sqlType = columnNameToTypeMap.get(columnName);
+ }
+ if (unresolvedColumnNames.size()>0) {
+ StringBuilder exceptionMessage = new StringBuilder();
+ boolean first = true;
+ exceptionMessage.append("Unable to resolve these column names to a single column family:\n");
+ for (String col : unresolvedColumnNames) {
+ if (first) first = false;
+ else exceptionMessage.append(",");
+ exceptionMessage.append(col);
+ }
+ exceptionMessage.append("\nAvailable columns with column families:\n");
+ first = true;
+ for (String col : fullColumnNameToTypeMap.keySet()) {
+ if (first) first = false;
+ else exceptionMessage.append(",");
+ exceptionMessage.append(col);
+ }
+ throw new SQLException(exceptionMessage.toString());
+ }
+
+ if (sqlType == null) {
+ if (strict) {
+ throw new SQLExceptionInfo.Builder(
+ SQLExceptionCode.COLUMN_NOT_FOUND)
+ .setColumnName(columnName)
+ .setTableName(tableName).build()
+ .buildException();
+ }
+ unfoundColumnCount++;
+ } else {
+ columnInfoList.add(new ColumnInfo(columnName, sqlType));
+ }
+ }
+ if (unfoundColumnCount == columns.size()) {
+ throw new SQLExceptionInfo.Builder(
+ SQLExceptionCode.COLUMN_NOT_FOUND)
+ .setColumnName(
+ Arrays.toString(columns.toArray(new String[0])))
+ .setTableName(tableName).build().buildException();
+ }
+ }
+ return columnInfoList;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/util/UpsertExecutor.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpsertExecutor.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpsertExecutor.java
new file mode 100644
index 0000000..d9ce5f2
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpsertExecutor.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.List;
+
+import org.apache.phoenix.schema.types.PDataType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+
+/**
+ * Executes upsert statements on a provided {@code PreparedStatement} based on incoming
+ * {@code RECORDS}. An {@link UpsertListener} is notified each time the prepared statement
+ * is executed.
+ */
+public abstract class UpsertExecutor<RECORD, FIELD> implements Closeable {
+
+ /**
+ * A listener that is called for events based on incoming JSON data.
+ */
+ public interface UpsertListener<RECORD> {
+
+ /**
+ * Called when an upsert has been sucessfully completed. The given upsertCount is the total number of upserts
+ * completed on the caller up to this point.
+ *
+ * @param upsertCount total number of upserts that have been completed
+ */
+ void upsertDone(long upsertCount);
+
+
+ /**
+ * Called when executing a prepared statement has failed on a given record.
+ *
+ * @param record the JSON record that was being upserted when the error occurred
+ */
+ void errorOnRecord(RECORD record, Throwable throwable);
+ }
+
+ private static final Logger LOG = LoggerFactory.getLogger(UpsertExecutor.class);
+
+ protected final Connection conn;
+ protected final List<ColumnInfo> columnInfos;
+ protected final List<PDataType> dataTypes;
+ protected final List<Function<FIELD, Object>> conversionFunctions;
+ protected final PreparedStatement preparedStatement;
+ protected final UpsertListener<RECORD> upsertListener;
+ protected long upsertCount = 0L;
+ protected boolean initFinished = false; // allow subclasses to finish initialization
+
+ private static PreparedStatement createStatement(Connection conn, String tableName,
+ List<ColumnInfo> columnInfoList) {
+ PreparedStatement preparedStatement;
+ try {
+ String upsertSql = QueryUtil.constructUpsertStatement(tableName, columnInfoList);
+ LOG.info("Upserting SQL data with {}", upsertSql);
+ preparedStatement = conn.prepareStatement(upsertSql);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ return preparedStatement;
+ }
+
+ /**
+ * Construct with the definition of incoming columns, and the statement upon which upsert
+ * statements are to be performed.
+ */
+ public UpsertExecutor(Connection conn, String tableName,
+ List<ColumnInfo> columnInfoList, UpsertListener<RECORD> upsertListener) {
+ this(conn, columnInfoList, createStatement(conn, tableName, columnInfoList), upsertListener);
+ }
+
+ /** Testing constructor. Do not use in prod. */
+ @VisibleForTesting
+ protected UpsertExecutor(Connection conn, List<ColumnInfo> columnInfoList,
+ PreparedStatement preparedStatement, UpsertListener<RECORD> upsertListener) {
+ this.conn = conn;
+ this.upsertListener = upsertListener;
+ this.columnInfos = columnInfoList;
+ this.preparedStatement = preparedStatement;
+ this.dataTypes = Lists.newArrayList();
+ this.conversionFunctions = Lists.newArrayList();
+ }
+
+ /**
+ * Awkward protocol allows subclass constructors to finish initializing context before
+ * proceeding to record processing.
+ */
+ protected void finishInit() {
+ for (ColumnInfo columnInfo : columnInfos) {
+ PDataType dataType = PDataType.fromTypeId(columnInfo.getSqlType());
+ dataTypes.add(dataType);
+ conversionFunctions.add(createConversionFunction(dataType));
+ }
+ this.initFinished = true;
+ }
+
+ /**
+ * Execute upserts for each JSON record contained in the given iterable, notifying this instance's
+ * {@code UpsertListener} for each completed upsert.
+ *
+ * @param records iterable of JSON records to be upserted
+ */
+ public void execute(Iterable<RECORD> records) {
+ if (!initFinished) {
+ finishInit();
+ }
+ for (RECORD record : records) {
+ execute(record);
+ }
+ }
+
+ /**
+ * Upsert a single record.
+ *
+ * @param record JSON record containing the data to be upserted
+ */
+ protected abstract void execute(RECORD record);
+
+ @Override
+ public void close() throws IOException {
+ try {
+ preparedStatement.close();
+ } catch (SQLException e) {
+ // An exception while closing the prepared statement is most likely a sign of a real problem, so we don't
+ // want to hide it with closeQuietly or something similar
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected abstract Function<FIELD, Object> createConversionFunction(PDataType dataType);
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/util/csv/CsvUpsertExecutor.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/csv/CsvUpsertExecutor.java b/phoenix-core/src/main/java/org/apache/phoenix/util/csv/CsvUpsertExecutor.java
index e680f5c..4a3af21 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/csv/CsvUpsertExecutor.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/csv/CsvUpsertExecutor.java
@@ -17,132 +17,55 @@
*/
package org.apache.phoenix.util.csv;
-import java.io.Closeable;
-import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Types;
import java.util.List;
import java.util.Properties;
-
import javax.annotation.Nullable;
import org.apache.commons.csv.CSVRecord;
-import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PTimestamp;
import org.apache.phoenix.util.ColumnInfo;
import org.apache.phoenix.util.DateUtil;
-import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.UpsertExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-/**
- * Executes upsert statements on a provided {@code PreparedStatement} based on incoming CSV records, notifying a
- * listener each time the prepared statement is executed.
- */
-public class CsvUpsertExecutor implements Closeable {
+/** {@link UpsertExecutor} over {@link CSVRecord}s. */
+public class CsvUpsertExecutor extends UpsertExecutor<CSVRecord, String> {
private static final Logger LOG = LoggerFactory.getLogger(CsvUpsertExecutor.class);
- private final String arrayElementSeparator;
- private final Connection conn;
- private final List<PDataType> dataTypes;
- private final List<Function<String,Object>> conversionFunctions;
- private final PreparedStatement preparedStatement;
- private final UpsertListener upsertListener;
- private long upsertCount = 0L;
-
- /**
- * A listener that is called for events based on incoming CSV data.
- */
- public static interface UpsertListener {
-
- /**
- * Called when an upsert has been sucessfully completed. The given upsertCount is the total number of upserts
- * completed on the caller up to this point.
- *
- * @param upsertCount total number of upserts that have been completed
- */
- void upsertDone(long upsertCount);
-
-
- /**
- * Called when executing a prepared statement has failed on a given record.
- *
- * @param csvRecord the CSV record that was being upserted when the error occurred
- */
- void errorOnRecord(CSVRecord csvRecord, Throwable throwable);
- }
-
-
- /**
- * Static constructor method for creating a CsvUpsertExecutor.
- *
- * @param conn Phoenix connection upon which upserts are to be performed
- * @param tableName name of the table in which upserts are to be performed
- * @param columnInfoList description of the columns to be upserted to, in the same order as in the CSV input
- * @param upsertListener listener that will be notified of upserts, can be null
- * @param arrayElementSeparator separator string to delimit string representations of arrays
- * @return the created CsvUpsertExecutor
- */
- public static CsvUpsertExecutor create(PhoenixConnection conn, String tableName, List<ColumnInfo> columnInfoList,
- UpsertListener upsertListener, String arrayElementSeparator) {
- PreparedStatement preparedStatement = null;
- try {
- String upsertSql = QueryUtil.constructUpsertStatement(tableName, columnInfoList);
- LOG.info("Upserting SQL data with {}", upsertSql);
- preparedStatement = conn.prepareStatement(upsertSql);
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- return new CsvUpsertExecutor(conn, columnInfoList, preparedStatement, upsertListener,
- arrayElementSeparator);
- }
+ protected final String arrayElementSeparator;
- /**
- * Construct with the definition of incoming columns, and the statement upon which upsert statements
- * are to be performed.
- */
- CsvUpsertExecutor(Connection conn, List<ColumnInfo> columnInfoList, PreparedStatement preparedStatement,
- UpsertListener upsertListener, String arrayElementSeparator) {
- this.conn = conn;
- this.preparedStatement = preparedStatement;
- this.upsertListener = upsertListener;
+ /** Testing constructor. Do not use in prod. */
+ @VisibleForTesting
+ protected CsvUpsertExecutor(Connection conn, List<ColumnInfo> columnInfoList,
+ PreparedStatement stmt, UpsertListener<CSVRecord> upsertListener,
+ String arrayElementSeparator) {
+ super(conn, columnInfoList, stmt, upsertListener);
this.arrayElementSeparator = arrayElementSeparator;
- this.dataTypes = Lists.newArrayList();
- this.conversionFunctions = Lists.newArrayList();
- for (ColumnInfo columnInfo : columnInfoList) {
- PDataType dataType = PDataType.fromTypeId(columnInfo.getSqlType());
- dataTypes.add(dataType);
- conversionFunctions.add(createConversionFunction(dataType));
- }
+ finishInit();
}
- /**
- * Execute upserts for each CSV record contained in the given iterable, notifying this instance's
- * {@code UpsertListener} for each completed upsert.
- *
- * @param csvRecords iterable of CSV records to be upserted
- */
- public void execute(Iterable<CSVRecord> csvRecords) {
- for (CSVRecord csvRecord : csvRecords) {
- execute(csvRecord);
- }
+ public CsvUpsertExecutor(Connection conn, String tableName,
+ List<ColumnInfo> columnInfoList, UpsertListener<CSVRecord> upsertListener,
+ String arrayElementSeparator) {
+ super(conn, tableName, columnInfoList, upsertListener);
+ this.arrayElementSeparator = arrayElementSeparator;
+ finishInit();
}
- /**
- * Upsert a single record.
- *
- * @param csvRecord CSV record containing the data to be upserted
- */
- void execute(CSVRecord csvRecord) {
+ @Override
+ protected void execute(CSVRecord csvRecord) {
try {
if (csvRecord.size() < conversionFunctions.size()) {
String message = String.format("CSV record does not have enough values (has %d, but needs %d)",
@@ -170,17 +93,7 @@ public class CsvUpsertExecutor implements Closeable {
}
@Override
- public void close() throws IOException {
- try {
- preparedStatement.close();
- } catch (SQLException e) {
- // An exception while closing the prepared statement is most likely a sign of a real problem, so we don't
- // want to hide it with closeQuietly or something similar
- throw new RuntimeException(e);
- }
- }
-
- private Function<String, Object> createConversionFunction(PDataType dataType) {
+ protected Function<String, Object> createConversionFunction(PDataType dataType) {
if (dataType.isArrayType()) {
return new ArrayDatatypeConversionFunction(
new StringToArrayConverter(
@@ -201,7 +114,7 @@ public class CsvUpsertExecutor implements Closeable {
private final DateUtil.DateTimeParser dateTimeParser;
SimpleDatatypeConversionFunction(PDataType dataType, Connection conn) {
- Properties props = null;
+ Properties props;
try {
props = conn.getClientInfo();
} catch (SQLException e) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/util/json/JsonUpsertExecutor.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/json/JsonUpsertExecutor.java b/phoenix-core/src/main/java/org/apache/phoenix/util/json/JsonUpsertExecutor.java
new file mode 100644
index 0000000..bbe0e30
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/json/JsonUpsertExecutor.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util.json;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import javax.annotation.Nullable;
+
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PTimestamp;
+import org.apache.phoenix.util.ColumnInfo;
+import org.apache.phoenix.util.DateUtil;
+import org.apache.phoenix.util.UpsertExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.CaseFormat;
+import com.google.common.base.Function;
+
+/** {@link UpsertExecutor} over {@link Map} objects, as parsed from JSON. */
+public class JsonUpsertExecutor extends UpsertExecutor<Map<?, ?>, Object> {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(JsonUpsertExecutor.class);
+
+ /** Testing constructor. Do not use in prod. */
+ @VisibleForTesting
+ protected JsonUpsertExecutor(Connection conn, List<ColumnInfo> columnInfoList,
+ PreparedStatement stmt, UpsertListener<Map<?, ?>> upsertListener) {
+ super(conn, columnInfoList, stmt, upsertListener);
+ finishInit();
+ }
+
+ public JsonUpsertExecutor(Connection conn, String tableName, List<ColumnInfo> columnInfoList,
+ UpsertExecutor.UpsertListener<Map<?, ?>> upsertListener) {
+ super(conn, tableName, columnInfoList, upsertListener);
+ finishInit();
+ }
+
+ @Override
+ protected void execute(Map<?, ?> record) {
+ int fieldIndex = 0;
+ String colName = null;
+ try {
+ if (record.size() < conversionFunctions.size()) {
+ String message = String.format("JSON record does not have enough values (has %d, but needs %d)",
+ record.size(), conversionFunctions.size());
+ throw new IllegalArgumentException(message);
+ }
+ for (fieldIndex = 0; fieldIndex < conversionFunctions.size(); fieldIndex++) {
+ colName = CaseFormat.UPPER_UNDERSCORE.to(
+ CaseFormat.LOWER_UNDERSCORE, columnInfos.get(fieldIndex).getColumnName());
+ if (colName.contains(".")) {
+ StringBuilder sb = new StringBuilder();
+ String[] parts = colName.split("\\.");
+ // assume first part is the column family name; omita
+ for (int i = 1; i < parts.length; i++) {
+ sb.append(parts[i]);
+ if (i != parts.length - 1) {
+ sb.append(".");
+ }
+ }
+ colName = sb.toString();
+ }
+ if (colName.contains("\"")) {
+ colName = colName.replace("\"", "");
+ }
+ Object sqlValue = conversionFunctions.get(fieldIndex).apply(record.get(colName));
+ if (sqlValue != null) {
+ preparedStatement.setObject(fieldIndex + 1, sqlValue);
+ } else {
+ preparedStatement.setNull(fieldIndex + 1, dataTypes.get(fieldIndex).getSqlType());
+ }
+ }
+ preparedStatement.execute();
+ upsertListener.upsertDone(++upsertCount);
+ } catch (Exception e) {
+ if (LOG.isDebugEnabled()) {
+ // Even though this is an error we only log it with debug logging because we're notifying the
+ // listener, and it can do its own logging if needed
+ LOG.debug("Error on record " + record + ", fieldIndex " + fieldIndex + ", colName " + colName, e);
+ }
+ upsertListener.errorOnRecord(record, new Exception("fieldIndex: " + fieldIndex + ", colName " + colName, e));
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ preparedStatement.close();
+ } catch (SQLException e) {
+ // An exception while closing the prepared statement is most likely a sign of a real problem, so we don't
+ // want to hide it with closeQuietly or something similar
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ protected Function<Object, Object> createConversionFunction(PDataType dataType) {
+ if (dataType.isArrayType()) {
+ return new ArrayDatatypeConversionFunction(
+ new ObjectToArrayConverter(
+ conn,
+ PDataType.fromTypeId(dataType.getSqlType() - PDataType.ARRAY_TYPE_BASE)));
+ } else {
+ return new SimpleDatatypeConversionFunction(dataType, this.conn);
+ }
+ }
+
+ /**
+ * Performs typed conversion from String values to a given column value type.
+ */
+ static class SimpleDatatypeConversionFunction implements Function<Object, Object> {
+
+ private final PDataType dataType;
+ private final DateUtil.DateTimeParser dateTimeParser;
+
+ SimpleDatatypeConversionFunction(PDataType dataType, Connection conn) {
+ Properties props;
+ try {
+ props = conn.getClientInfo();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ this.dataType = dataType;
+ if (dataType.isCoercibleTo(PTimestamp.INSTANCE)) {
+ // TODO: move to DateUtil
+ String dateFormat;
+ int dateSqlType = dataType.getResultSetSqlType();
+ if (dateSqlType == Types.DATE) {
+ dateFormat = props.getProperty(QueryServices.DATE_FORMAT_ATTRIB,
+ DateUtil.DEFAULT_DATE_FORMAT);
+ } else if (dateSqlType == Types.TIME) {
+ dateFormat = props.getProperty(QueryServices.TIME_FORMAT_ATTRIB,
+ DateUtil.DEFAULT_TIME_FORMAT);
+ } else {
+ dateFormat = props.getProperty(QueryServices.TIMESTAMP_FORMAT_ATTRIB,
+ DateUtil.DEFAULT_TIMESTAMP_FORMAT);
+ }
+ String timeZoneId = props.getProperty(QueryServices.DATE_FORMAT_TIMEZONE_ATTRIB,
+ QueryServicesOptions.DEFAULT_DATE_FORMAT_TIMEZONE);
+ this.dateTimeParser = DateUtil.getDateTimeParser(dateFormat, dataType, timeZoneId);
+ } else {
+ this.dateTimeParser = null;
+ }
+ }
+
+ @Nullable
+ @Override
+ public Object apply(@Nullable Object input) {
+ if (input == null) {
+ return null;
+ }
+ if (dateTimeParser != null && input instanceof String) {
+ final String s = (String) input;
+ long epochTime = dateTimeParser.parseDateTime(s);
+ byte[] byteValue = new byte[dataType.getByteSize()];
+ dataType.getCodec().encodeLong(epochTime, byteValue, 0);
+ return dataType.toObject(byteValue);
+ }
+ return dataType.toObject(input, dataType);
+ }
+ }
+
+ /**
+ * Converts string representations of arrays into Phoenix arrays of the correct type.
+ */
+ private static class ArrayDatatypeConversionFunction implements Function<Object, Object> {
+
+ private final ObjectToArrayConverter arrayConverter;
+
+ private ArrayDatatypeConversionFunction(ObjectToArrayConverter arrayConverter) {
+ this.arrayConverter = arrayConverter;
+ }
+
+ @Nullable
+ @Override
+ public Object apply(@Nullable Object input) {
+ try {
+ return arrayConverter.toArray(input);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/main/java/org/apache/phoenix/util/json/ObjectToArrayConverter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/json/ObjectToArrayConverter.java b/phoenix-core/src/main/java/org/apache/phoenix/util/json/ObjectToArrayConverter.java
new file mode 100644
index 0000000..16bef15
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/json/ObjectToArrayConverter.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util.json;
+
+import java.sql.Array;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.List;
+
+import org.apache.phoenix.schema.types.PDataType;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+
+/**
+ * Converts Objects (presumably lists) into Phoenix arrays.
+ */
+class ObjectToArrayConverter {
+
+ private final Connection conn;
+ private final PDataType elementDataType;
+ private final JsonUpsertExecutor.SimpleDatatypeConversionFunction elementConvertFunction;
+
+ /**
+ * Instantiate with the array value separator and data type.
+ *
+ * @param conn Phoenix connection to target database
+ * @param elementDataType datatype of the elements of arrays to be created
+ */
+ public ObjectToArrayConverter(Connection conn, PDataType elementDataType) {
+ this.conn = conn;
+ this.elementDataType = elementDataType;
+ this.elementConvertFunction =
+ new JsonUpsertExecutor.SimpleDatatypeConversionFunction(elementDataType, this.conn);
+ }
+
+ /**
+ * Convert an input delimited string into a phoenix array of the configured type.
+ *
+ * @param input string containing delimited array values
+ * @return the array containing the values represented in the input string
+ */
+ public Array toArray(Object input) throws SQLException {
+ if (input == null) {
+ return conn.createArrayOf(elementDataType.getSqlTypeName(), new Object[0]);
+ }
+ List<?> list = (List<?>) input;
+ if (list.isEmpty()) {
+ return conn.createArrayOf(elementDataType.getSqlTypeName(), new Object[0]);
+ }
+ return conn.createArrayOf(elementDataType.getSqlTypeName(),
+ Lists.newArrayList(Iterables.transform(list, elementConvertFunction)).toArray());
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/BulkLoadToolTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/BulkLoadToolTest.java b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/BulkLoadToolTest.java
new file mode 100644
index 0000000..95e9b43
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/BulkLoadToolTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.commons.cli.CommandLine;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static org.junit.Assert.assertEquals;
+
+@RunWith(Parameterized.class)
+public class BulkLoadToolTest {
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> params() {
+ return Arrays.asList(new Object[][]{
+ { new CsvBulkLoadTool() },
+ { new JsonBulkLoadTool() },
+ });
+ }
+
+ @Parameterized.Parameter(value = 0)
+ public AbstractBulkLoadTool bulkLoadTool;
+
+ @Test
+ public void testParseOptions() {
+ CommandLine cmdLine = bulkLoadTool.parseOptions(new String[] { "--input", "/input",
+ "--table", "mytable" });
+
+ assertEquals("mytable", cmdLine.getOptionValue(CsvBulkLoadTool.TABLE_NAME_OPT.getOpt()));
+ assertEquals("/input", cmdLine.getOptionValue(CsvBulkLoadTool.INPUT_PATH_OPT.getOpt()));
+ }
+
+ @Test(expected=IllegalStateException.class)
+ public void testParseOptions_ExtraArguments() {
+ bulkLoadTool.parseOptions(new String[] { "--input", "/input",
+ "--table", "mytable", "these", "shouldnt", "be", "here" });
+ }
+
+ @Test(expected=IllegalStateException.class)
+ public void testParseOptions_NoInput() {
+ bulkLoadTool.parseOptions(new String[] { "--table", "mytable" });
+ }
+
+ @Test(expected=IllegalStateException.class)
+ public void testParseOptions_NoTable() {
+ bulkLoadTool.parseOptions(new String[] { "--input", "/input" });
+ }
+
+ @Test
+ public void testGetQualifiedTableName() {
+ assertEquals("MYSCHEMA.MYTABLE", CsvBulkLoadTool.getQualifiedTableName("mySchema", "myTable"));
+ }
+
+ @Test
+ public void testGetQualifiedTableName_NullSchema() {
+ assertEquals("MYTABLE", CsvBulkLoadTool.getQualifiedTableName(null, "myTable"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvBulkImportUtilTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvBulkImportUtilTest.java b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvBulkImportUtilTest.java
index f52a837..3c6271a 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvBulkImportUtilTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvBulkImportUtilTest.java
@@ -17,9 +17,6 @@
*/
package org.apache.phoenix.mapreduce;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -29,11 +26,10 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
-import org.apache.phoenix.schema.types.PInteger;
-import org.apache.phoenix.util.ColumnInfo;
import org.junit.Test;
-import com.google.common.collect.ImmutableList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
public class CsvBulkImportUtilTest {
@@ -41,16 +37,11 @@ public class CsvBulkImportUtilTest {
public void testInitCsvImportJob() throws IOException {
Configuration conf = new Configuration();
- String tableName = "SCHEMANAME.TABLENAME";
char delimiter = '\001';
char quote = '\002';
char escape = '!';
- List<ColumnInfo> columnInfoList = ImmutableList.of(
- new ColumnInfo("MYCOL", PInteger.INSTANCE.getSqlType()));
-
- CsvBulkImportUtil.initCsvImportJob(
- conf, tableName, delimiter, quote, escape, null, columnInfoList, true);
+ CsvBulkImportUtil.initCsvImportJob(conf, delimiter, quote, escape, null);
// Serialize and deserialize the config to ensure that there aren't any issues
// with non-printable characters as delimiters
@@ -61,7 +52,6 @@ public class CsvBulkImportUtilTest {
Configuration deserialized = new Configuration();
deserialized.addResource(new FileInputStream(tempFile));
- assertEquals(tableName, deserialized.get(CsvToKeyValueMapper.TABLE_NAME_CONFKEY));
assertEquals(Character.valueOf('\001'),
CsvBulkImportUtil.getCharacter(deserialized, CsvToKeyValueMapper.FIELD_DELIMITER_CONFKEY));
assertEquals(Character.valueOf('\002'),
@@ -69,8 +59,6 @@ public class CsvBulkImportUtilTest {
assertEquals(Character.valueOf('!'),
CsvBulkImportUtil.getCharacter(deserialized, CsvToKeyValueMapper.ESCAPE_CHAR_CONFKEY));
assertNull(deserialized.get(CsvToKeyValueMapper.ARRAY_DELIMITER_CONFKEY));
- assertEquals(columnInfoList, CsvToKeyValueMapper.buildColumnInfoList(deserialized));
- assertEquals(true, deserialized.getBoolean(CsvToKeyValueMapper.IGNORE_INVALID_ROW_CONFKEY, false));
tempFile.delete();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolTest.java b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolTest.java
deleted file mode 100644
index 33bb976..0000000
--- a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.mapreduce;
-
-import org.apache.commons.cli.CommandLine;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class CsvBulkLoadToolTest {
-
- private CsvBulkLoadTool bulkLoadTool;
-
- @Before
- public void setUp() {
- bulkLoadTool = new CsvBulkLoadTool();
- }
-
- @Test
- public void testParseOptions() {
- CommandLine cmdLine = bulkLoadTool.parseOptions(new String[] { "--input", "/input",
- "--table", "mytable" });
-
- assertEquals("mytable", cmdLine.getOptionValue(CsvBulkLoadTool.TABLE_NAME_OPT.getOpt()));
- assertEquals("/input", cmdLine.getOptionValue(CsvBulkLoadTool.INPUT_PATH_OPT.getOpt()));
- }
-
- @Test(expected=IllegalStateException.class)
- public void testParseOptions_ExtraArguments() {
- bulkLoadTool.parseOptions(new String[] { "--input", "/input",
- "--table", "mytable", "these", "shouldnt", "be", "here" });
- }
-
- @Test(expected=IllegalStateException.class)
- public void testParseOptions_NoInput() {
- bulkLoadTool.parseOptions(new String[] { "--table", "mytable" });
- }
-
- @Test(expected=IllegalStateException.class)
- public void testParseOptions_NoTable() {
- bulkLoadTool.parseOptions(new String[] { "--input", "/input" });
- }
-
- @Test
- public void testGetQualifiedTableName() {
- assertEquals("MYSCHEMA.MYTABLE", CsvBulkLoadTool.getQualifiedTableName("mySchema", "myTable"));
- }
-
- @Test
- public void testGetQualifiedTableName_NullSchema() {
- assertEquals("MYTABLE", CsvBulkLoadTool.getQualifiedTableName(null, "myTable"));
- }
-}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapperTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapperTest.java b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapperTest.java
index dc6f497..fe4e068 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapperTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapperTest.java
@@ -17,25 +17,13 @@
*/
package org.apache.phoenix.mapreduce;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
import java.io.IOException;
-import java.util.List;
import org.apache.commons.csv.CSVRecord;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
-import org.apache.phoenix.schema.types.PInteger;
-import org.apache.phoenix.schema.types.PIntegerArray;
-import org.apache.phoenix.schema.types.PUnsignedInt;
-import org.apache.phoenix.util.ColumnInfo;
import org.junit.Test;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
public class CsvToKeyValueMapperTest {
@@ -62,72 +50,4 @@ public class CsvToKeyValueMapperTest {
assertTrue(parsed.isConsistent());
assertEquals(1, parsed.getRecordNumber());
}
-
-
- @Test
- public void testBuildColumnInfoList() {
- List<ColumnInfo> columnInfoList = ImmutableList.of(
- new ColumnInfo("idCol", PInteger.INSTANCE.getSqlType()),
- new ColumnInfo("unsignedIntCol", PUnsignedInt.INSTANCE.getSqlType()),
- new ColumnInfo("stringArrayCol", PIntegerArray.INSTANCE.getSqlType()));
-
- Configuration conf = new Configuration();
- CsvToKeyValueMapper.configureColumnInfoList(conf, columnInfoList);
- List<ColumnInfo> fromConfig = CsvToKeyValueMapper.buildColumnInfoList(conf);
-
- assertEquals(columnInfoList, fromConfig);
- }
-
- @Test
- public void testBuildColumnInfoList_ContainingNulls() {
- // A null value in the column info list means "skip that column in the input"
- List<ColumnInfo> columnInfoListWithNull = Lists.newArrayList(
- new ColumnInfo("idCol", PInteger.INSTANCE.getSqlType()),
- null,
- new ColumnInfo("unsignedIntCol", PUnsignedInt.INSTANCE.getSqlType()),
- new ColumnInfo("stringArrayCol", PIntegerArray.INSTANCE.getSqlType()));
-
- Configuration conf = new Configuration();
- CsvToKeyValueMapper.configureColumnInfoList(conf, columnInfoListWithNull);
- List<ColumnInfo> fromConfig = CsvToKeyValueMapper.buildColumnInfoList(conf);
-
- assertEquals(columnInfoListWithNull, fromConfig);
- }
-
- @Test
- public void testLoadPreUpdateProcessor() {
- Configuration conf = new Configuration();
- conf.setClass(PhoenixConfigurationUtil.UPSERT_HOOK_CLASS_CONFKEY, MockUpsertProcessor.class,
- ImportPreUpsertKeyValueProcessor.class);
-
- ImportPreUpsertKeyValueProcessor processor = PhoenixConfigurationUtil.loadPreUpsertProcessor(conf);
- assertEquals(MockUpsertProcessor.class, processor.getClass());
- }
-
- @Test
- public void testLoadPreUpdateProcessor_NotConfigured() {
-
- Configuration conf = new Configuration();
- ImportPreUpsertKeyValueProcessor processor = PhoenixConfigurationUtil.loadPreUpsertProcessor(conf);
-
- assertEquals(CsvToKeyValueMapper.DefaultImportPreUpsertKeyValueProcessor.class,
- processor.getClass());
- }
-
- @Test(expected=IllegalStateException.class)
- public void testLoadPreUpdateProcessor_ClassNotFound() {
- Configuration conf = new Configuration();
- conf.set(PhoenixConfigurationUtil.UPSERT_HOOK_CLASS_CONFKEY, "MyUndefinedClass");
-
- PhoenixConfigurationUtil.loadPreUpsertProcessor(conf);
- }
-
-
- static class MockUpsertProcessor implements ImportPreUpsertKeyValueProcessor {
-
- @Override
- public List<KeyValue> preUpsert(byte[] rowKey, List<KeyValue> keyValues) {
- throw new UnsupportedOperationException("Not yet implemented");
- }
- }
}
[42/52] [abbrv] phoenix git commit: PHOENIX-2356 Set TRANSACTIONAL to
true for derived views when transitioning to transactional table
Posted by ma...@apache.org.
PHOENIX-2356 Set TRANSACTIONAL to true for derived views when transitioning to transactional table
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/93acc278
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/93acc278
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/93acc278
Branch: refs/heads/calcite
Commit: 93acc2788049b12ecda4b962dfdde9f0b6ed4fed
Parents: a8238f9
Author: Thomas D'Silva <td...@salesforce.com>
Authored: Fri Dec 4 12:09:04 2015 -0800
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Mon Dec 7 12:04:59 2015 -0800
----------------------------------------------------------------------
.../phoenix/end2end/AlterTableWithViewsIT.java | 48 +++++++++++++++++++-
.../coprocessor/MetaDataEndpointImpl.java | 43 +++++++++++-------
2 files changed, 73 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/93acc278/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
index 3f311ea..e3d78ea 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
@@ -34,9 +34,11 @@ import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.compile.QueryPlan;
import org.apache.phoenix.exception.SQLExceptionCode;
@@ -44,19 +46,32 @@ import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
-import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
+import co.cask.tephra.hbase11.coprocessor.TransactionProcessor;
+
import com.google.common.base.Objects;
+import com.google.common.collect.Maps;
public class AlterTableWithViewsIT extends BaseHBaseManagedTimeIT {
+
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(1);
+ props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
@Test
public void testAddNewColumnsToBaseTableWithViews() throws Exception {
@@ -1096,6 +1111,37 @@ public class AlterTableWithViewsIT extends BaseHBaseManagedTimeIT {
}
}
+ @Test
+ public void testMakeBaseTableTransactional() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.createStatement().execute("CREATE TABLE IF NOT EXISTS TABLEWITHVIEW ("
+ + " ID char(1) NOT NULL,"
+ + " COL1 integer NOT NULL,"
+ + " COL2 bigint NOT NULL,"
+ + " CONSTRAINT NAME_PK PRIMARY KEY (ID, COL1, COL2)"
+ + " )");
+ assertTableDefinition(conn, "TABLEWITHVIEW", PTableType.TABLE, null, 0, 3, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, "ID", "COL1", "COL2");
+
+ conn.createStatement().execute("CREATE VIEW VIEWOFTABLE ( VIEW_COL1 DECIMAL(10,2), VIEW_COL2 VARCHAR ) AS SELECT * FROM TABLEWITHVIEW");
+ assertTableDefinition(conn, "VIEWOFTABLE", PTableType.VIEW, "TABLEWITHVIEW", 0, 5, 3, "ID", "COL1", "COL2", "VIEW_COL1", "VIEW_COL2");
+
+ HTableInterface htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("TABLEWITHVIEW"));
+ assertFalse(htable.getTableDescriptor().getCoprocessors().contains(TransactionProcessor.class.getName()));
+ assertFalse(conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "TABLEWITHVIEW")).isTransactional());
+ assertFalse(conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "VIEWOFTABLE")).isTransactional());
+
+ // make the base table transactional
+ conn.createStatement().execute("ALTER TABLE TABLEWITHVIEW SET TRANSACTIONAL=true");
+ // query the view to force the table cache to be updated
+ conn.createStatement().execute("SELECT * FROM VIEWOFTABLE");
+ htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("TABLEWITHVIEW"));
+ assertTrue(htable.getTableDescriptor().getCoprocessors().contains(TransactionProcessor.class.getName()));
+ assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "TABLEWITHVIEW")).isTransactional());
+ assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "VIEWOFTABLE")).isTransactional());
+ }
+
+ }
+
private static long getTableSequenceNumber(PhoenixConnection conn, String tableName) throws SQLException {
PTable table = conn.getTable(new PTableKey(conn.getTenantId(), SchemaUtil.normalizeIdentifier(tableName)));
return table.getSequenceNumber();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/93acc278/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 7c80f23..f997149 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -1805,7 +1805,22 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
return columnKey;
}
- MetaDataMutationResult addRowsToChildViews(PTable basePhysicalTable, List<Mutation> tableMetadata, List<Mutation> mutationsForAddingColumnsToViews, byte[] schemaName, byte[] tableName,
+ private boolean switchAttribute(PTable table, boolean currAttribute, List<Mutation> tableMetaData, byte[] attrQualifier) {
+ for (Mutation m : tableMetaData) {
+ if (m instanceof Put) {
+ Put p = (Put)m;
+ List<Cell> cells = p.get(TABLE_FAMILY_BYTES, attrQualifier);
+ if (cells != null && cells.size() > 0) {
+ Cell cell = cells.get(0);
+ boolean newAttribute = (boolean)PBoolean.INSTANCE.toObject(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ return currAttribute != newAttribute;
+ }
+ }
+ }
+ return false;
+ }
+
+ private MetaDataMutationResult addRowsToChildViews(PTable basePhysicalTable, List<Mutation> tableMetadata, List<Mutation> mutationsForAddingColumnsToViews, byte[] schemaName, byte[] tableName,
List<ImmutableBytesPtr> invalidateList, long clientTimeStamp, TableViewFinderResult childViewsResult,
Region region, List<RowLock> locks) throws IOException, SQLException {
List<PutWithOrdinalPosition> columnPutsForBaseTable = new ArrayList<>(tableMetadata.size());
@@ -2043,6 +2058,15 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
if (view.rowKeyOrderOptimizable()) {
UpgradeUtil.addRowKeyOrderOptimizableCell(mutationsForAddingColumnsToViews, viewKey, clientTimeStamp);
}
+
+ // if switching from from non tx to tx
+ if (!basePhysicalTable.isTransactional() && switchAttribute(basePhysicalTable, basePhysicalTable.isTransactional(), tableMetadata, TRANSACTIONAL_BYTES)) {
+ invalidateList.add(new ImmutableBytesPtr(viewKey));
+ Put put = new Put(viewKey);
+ put.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ TRANSACTIONAL_BYTES, clientTimeStamp, PBoolean.INSTANCE.toBytes(true));
+ mutationsForAddingColumnsToViews.add(put);
+ }
}
return null;
}
@@ -2267,7 +2291,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
if (!childViewsResult.allViewsInSingleRegion()
|| table.getBaseColumnCount() == 0
|| !request.hasClientVersion()
- || switchTenancy(table, tableMetaData)) {
+ || switchAttribute(table, table.isMultiTenant(), tableMetaData, MULTI_TENANT_BYTES)) {
return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION,
EnvironmentEdgeManager.currentTimeMillis(), null);
} else {
@@ -2354,21 +2378,6 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
tableMetaData.addAll(mutationsForAddingColumnsToViews);
return null;
}
-
- private boolean switchTenancy(PTable table, List<Mutation> tableMetaData) {
- for (Mutation m : tableMetaData) {
- if (m instanceof Put) {
- Put p = (Put)m;
- List<Cell> cells = p.get(TABLE_FAMILY_BYTES, MULTI_TENANT_BYTES);
- if (cells != null && cells.size() > 0) {
- Cell cell = cells.get(0);
- boolean isMutlitenantProp = (boolean)PBoolean.INSTANCE.toObject(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
- return table.isMultiTenant() != isMutlitenantProp;
- }
- }
- }
- return false;
- }
});
if (result != null) {
done.run(MetaDataMutationResult.toProto(result));
[34/52] [abbrv] phoenix git commit: PHOENIX-2458 Refactoring tests
that start the txn manager to use BaseOwnClusterIT
Posted by ma...@apache.org.
PHOENIX-2458 Refactoring tests that start the txn manager to use BaseOwnClusterIT
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/0701e06d
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/0701e06d
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/0701e06d
Branch: refs/heads/calcite
Commit: 0701e06d48a90c372a073d6aac2c3259e5500bb6
Parents: 6aa4cc6
Author: Thomas D'Silva <td...@salesforce.com>
Authored: Tue Dec 1 13:58:49 2015 -0800
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Wed Dec 2 14:10:30 2015 -0800
----------------------------------------------------------------------
.../end2end/index/MutableIndexFailureIT.java | 92 +++++---------------
.../end2end/index/txn/TxWriteFailureIT.java | 49 +++--------
.../apache/phoenix/execute/PartialCommitIT.java | 41 +++------
.../apache/phoenix/rpc/PhoenixClientRpcIT.java | 1 +
.../java/org/apache/phoenix/query/BaseTest.java | 7 ++
5 files changed, 55 insertions(+), 135 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/0701e06d/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index 801d6c3..9032ee9 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -17,11 +17,6 @@
*/
package org.apache.phoenix.end2end.index;
-import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
-import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
-import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR;
-import static org.apache.phoenix.util.PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PARAM;
-import static org.apache.phoenix.util.TestUtil.LOCALHOST;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -34,6 +29,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -43,11 +39,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseCluster;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.ServerName;
@@ -55,14 +47,9 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.hadoop.hbase.master.LoadBalancer;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.end2end.BaseOwnClusterHBaseManagedTimeIT;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
-import org.apache.phoenix.hbase.index.balancer.IndexLoadBalancer;
-import org.apache.phoenix.hbase.index.master.IndexMasterObserver;
-import org.apache.phoenix.jdbc.PhoenixTestDriver;
-import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PIndexState;
import org.apache.phoenix.schema.PTableType;
@@ -73,9 +60,7 @@ import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.StringUtil;
import org.apache.phoenix.util.TestUtil;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
@@ -95,11 +80,7 @@ import com.google.common.collect.Maps;
@Category(NeedsOwnMiniClusterTest.class)
@RunWith(Parameterized.class)
-@Ignore // see PHOENIX-2458
-public class MutableIndexFailureIT extends BaseTest {
- private static final int NUM_SLAVES = 4;
- private static PhoenixTestDriver driver;
- private static HBaseTestingUtility util;
+public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT {
private Timer scheduleTimer;
private String tableName;
@@ -113,63 +94,34 @@ public class MutableIndexFailureIT extends BaseTest {
public MutableIndexFailureIT(boolean transactional) {
this.transactional = transactional;
this.tableDDLOptions = transactional ? " TRANSACTIONAL=true " : "";
- this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME;
+ this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + (transactional ? "_TXN" : "");
this.indexName = "IDX";
this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
this.fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
}
- @Before
- public void doSetup() throws Exception {
- Configuration conf = HBaseConfiguration.create();
- setUpConfigForMiniCluster(conf);
- conf.setInt("hbase.client.retries.number", 2);
- conf.setInt("hbase.client.pause", 5000);
- conf.setInt("hbase.balancer.period", Integer.MAX_VALUE);
- conf.setLong(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB, 0);
- conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName());
- conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, IndexLoadBalancer.class,
- LoadBalancer.class);
- util = new HBaseTestingUtility(conf);
- util.startMiniCluster(NUM_SLAVES);
- String clientPort = util.getConfiguration().get(QueryServices.ZOOKEEPER_PORT_ATTRIB);
- url = JDBC_PROTOCOL + JDBC_PROTOCOL_SEPARATOR + LOCALHOST + JDBC_PROTOCOL_SEPARATOR + clientPort
- + JDBC_PROTOCOL_TERMINATOR + PHOENIX_TEST_DRIVER_URL_PARAM;
- Map<String,String> props = Maps.newHashMapWithExpectedSize(1);
- props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
- driver = initAndRegisterDriver(url, new ReadOnlyProps(props.entrySet().iterator()));
- clusterInitialized = true;
- setupTxManager();
+ @BeforeClass
+ public static void doSetup() throws Exception {
+ Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(10);
+ serverProps.put("hbase.client.retries.number", "2");
+ serverProps.put("hbase.client.pause", "5000");
+ serverProps.put("hbase.balancer.period", String.valueOf(Integer.MAX_VALUE));
+ serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB, "0");
+ Map<String, String> clientProps = Collections.singletonMap(QueryServices.TRANSACTIONS_ENABLED, "true");
+ NUM_SLAVES_BASE = 4;
+ setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator()));
}
@Parameters(name = "transactional = {0}")
public static Collection<Boolean[]> data() {
return Arrays.asList(new Boolean[][] { { false }, { true } });
}
-
- @After
- public void tearDown() throws Exception {
- try {
- destroyDriver(driver);
- } finally {
- try {
- if(scheduleTimer != null){
- scheduleTimer.cancel();
- scheduleTimer = null;
- }
- } finally {
- util.shutdownMiniCluster();
- }
- }
- }
- @Ignore("See PHOENIX-2331")
@Test(timeout=300000)
public void testWriteFailureDisablesLocalIndex() throws Exception {
helpTestWriteFailureDisablesIndex(true);
}
- @Ignore("See PHOENIX-2332")
@Test(timeout=300000)
public void testWriteFailureDisablesIndex() throws Exception {
helpTestWriteFailureDisablesIndex(false);
@@ -219,7 +171,7 @@ public class MutableIndexFailureIT extends BaseTest {
TableName indexTable =
TableName.valueOf(localIndex ? MetaDataUtil
.getLocalIndexTableName(fullTableName) : fullIndexName);
- HBaseAdmin admin = this.util.getHBaseAdmin();
+ HBaseAdmin admin = this.getUtility().getHBaseAdmin();
HTableDescriptor indexTableDesc = admin.getTableDescriptor(indexTable);
try{
admin.disableTable(indexTable);
@@ -382,9 +334,9 @@ public class MutableIndexFailureIT extends BaseTest {
// find a RS which doesn't has CATALOG table
TableName catalogTable = TableName.valueOf("SYSTEM.CATALOG");
TableName indexTable = TableName.valueOf(fullIndexName);
- final HBaseCluster cluster = this.util.getHBaseCluster();
+ final HBaseCluster cluster = this.getUtility().getHBaseCluster();
Collection<ServerName> rss = cluster.getClusterStatus().getServers();
- HBaseAdmin admin = this.util.getHBaseAdmin();
+ HBaseAdmin admin = this.getUtility().getHBaseAdmin();
List<HRegionInfo> regions = admin.getTableRegions(catalogTable);
ServerName catalogRS = cluster.getServerHoldingRegion(regions.get(0).getTable(),
regions.get(0).getRegionName());
@@ -404,7 +356,7 @@ public class MutableIndexFailureIT extends BaseTest {
final HRegionInfo indexRegion = regions.get(0);
final ServerName dstRS = rsToBeKilled;
admin.move(indexRegion.getEncodedNameAsBytes(), Bytes.toBytes(rsToBeKilled.getServerName()));
- this.util.waitFor(30000, 200, new Waiter.Predicate<Exception>() {
+ this.getUtility().waitFor(30000, 200, new Waiter.Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
ServerName sn = cluster.getServerHoldingRegion(indexRegion.getTable(),
@@ -420,10 +372,10 @@ public class MutableIndexFailureIT extends BaseTest {
Thread.sleep(100);
// kill RS hosting index table
- this.util.getHBaseCluster().killRegionServer(rsToBeKilled);
+ this.getUtility().getHBaseCluster().killRegionServer(rsToBeKilled);
// wait for index table completes recovery
- this.util.waitUntilAllRegionsAssigned(indexTable);
+ this.getUtility().waitUntilAllRegionsAssigned(indexTable);
// Verify the metadata for index is correct.
do {
@@ -436,8 +388,6 @@ public class MutableIndexFailureIT extends BaseTest {
}
} while(true);
this.scheduleTimer.cancel();
-
- assertEquals(cluster.getClusterStatus().getDeadServers(), 1);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/0701e06d/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java
index 04bf1dc..8e36bf0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java
@@ -17,11 +17,6 @@
*/
package org.apache.phoenix.end2end.index.txn;
-import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
-import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
-import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR;
-import static org.apache.phoenix.util.PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PARAM;
-import static org.apache.phoenix.util.TestUtil.LOCALHOST;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -37,21 +32,17 @@ import java.util.Collection;
import java.util.Map;
import java.util.Properties;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseIOException;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.coprocessor.RegionObserver;
import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.end2end.BaseOwnClusterHBaseManagedTimeIT;
import org.apache.phoenix.hbase.index.Indexer;
-import org.apache.phoenix.jdbc.PhoenixTestDriver;
-import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
@@ -66,11 +57,8 @@ import org.junit.runners.Parameterized.Parameters;
import com.google.common.collect.Maps;
@RunWith(Parameterized.class)
-public class TxWriteFailureIT extends BaseTest {
+public class TxWriteFailureIT extends BaseOwnClusterHBaseManagedTimeIT {
- private static PhoenixTestDriver driver;
- private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-
private static final String SCHEMA_NAME = "S";
private static final String DATA_TABLE_NAME = "T";
private static final String INDEX_TABLE_NAME = "I";
@@ -85,29 +73,18 @@ public class TxWriteFailureIT extends BaseTest {
this.localIndex = localIndex;
this.mutable = mutable;
}
-
+
@BeforeClass
- public static void setupCluster() throws Exception {
- Configuration conf = TEST_UTIL.getConfiguration();
- setUpConfigForMiniCluster(conf);
- conf.setClass("hbase.coprocessor.region.classes", FailingRegionObserver.class, RegionObserver.class);
- conf.setBoolean("hbase.coprocessor.abortonerror", false);
- conf.setBoolean(Indexer.CHECK_VERSION_CONF_KEY, false);
- TEST_UTIL.startMiniCluster();
- String clientPort = TEST_UTIL.getConfiguration().get(
- QueryServices.ZOOKEEPER_PORT_ATTRIB);
- url = JDBC_PROTOCOL + JDBC_PROTOCOL_SEPARATOR + LOCALHOST
- + JDBC_PROTOCOL_SEPARATOR + clientPort
- + JDBC_PROTOCOL_TERMINATOR + PHOENIX_TEST_DRIVER_URL_PARAM;
-
- Map<String, String> props = Maps.newHashMapWithExpectedSize(2);
- // Must update config before starting server
- props.put(QueryServices.DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB, Boolean.toString(true));
- props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
- driver = initAndRegisterDriver(url, new ReadOnlyProps(props.entrySet().iterator()));
- clusterInitialized = true;
- setupTxManager();
- }
+ public static void doSetup() throws Exception {
+ Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(3);
+ serverProps.put("hbase.coprocessor.region.classes", FailingRegionObserver.class.getName());
+ serverProps.put("hbase.coprocessor.abortonerror", "false");
+ serverProps.put(Indexer.CHECK_VERSION_CONF_KEY, "false");
+ Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(10);
+ clientProps.put(QueryServices.DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB, "true");
+ clientProps.put(QueryServices.TRANSACTIONS_ENABLED, "true");
+ setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator()));
+ }
@Parameters(name="localIndex = {0} , mutable = {1}")
public static Collection<Boolean[]> data() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/0701e06d/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java b/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
index 4b06834..61aae62 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
@@ -22,29 +22,23 @@ package org.apache.phoenix.execute;
import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Sets.newHashSet;
import static java.util.Collections.singletonList;
-import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
-import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
-import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR;
-import static org.apache.phoenix.util.PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PARAM;
-import static org.apache.phoenix.util.TestUtil.LOCALHOST;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.sql.Connection;
-import java.sql.Driver;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -54,14 +48,13 @@ import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.coprocessor.RegionObserver;
import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.end2end.BaseOwnClusterIT;
import org.apache.phoenix.hbase.index.Indexer;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.jdbc.PhoenixConnection;
-import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.util.ReadOnlyProps;
@@ -76,7 +69,9 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
@RunWith(Parameterized.class)
-public class PartialCommitIT extends BaseTest {
+// Needs to extend BaseOwnClusterIT (and not BaseOwnClusterHBaseManagedTimeIT since table data
+// is created and populated once in the constructor
+public class PartialCommitIT extends BaseOwnClusterIT {
private final String A_SUCESS_TABLE;
private final String B_FAILURE_TABLE;
@@ -88,26 +83,16 @@ public class PartialCommitIT extends BaseTest {
private static final byte[] ROW_TO_FAIL_UPSERT_BYTES = Bytes.toBytes("fail me upsert");
private static final byte[] ROW_TO_FAIL_DELETE_BYTES = Bytes.toBytes("fail me delete");
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- private static Driver driver;
@BeforeClass
- public static void setupCluster() throws Exception {
- Configuration conf = TEST_UTIL.getConfiguration();
- setUpConfigForMiniCluster(conf);
- conf.setClass("hbase.coprocessor.region.classes", FailingRegionObserver.class, RegionObserver.class);
- conf.setBoolean("hbase.coprocessor.abortonerror", false);
- conf.setBoolean(Indexer.CHECK_VERSION_CONF_KEY, false);
- TEST_UTIL.startMiniCluster();
- String clientPort = TEST_UTIL.getConfiguration().get(QueryServices.ZOOKEEPER_PORT_ATTRIB);
- url = JDBC_PROTOCOL + JDBC_PROTOCOL_SEPARATOR + LOCALHOST + JDBC_PROTOCOL_SEPARATOR + clientPort
- + JDBC_PROTOCOL_TERMINATOR + PHOENIX_TEST_DRIVER_URL_PARAM;
-
- Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
- // Must update config before starting server
- driver = initAndRegisterDriver(url, ReadOnlyProps.EMPTY_PROPS);
- clusterInitialized = true;
- setupTxManager();
- createTablesWithABitOfData();
+ public static void doSetup() throws Exception {
+ Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(10);
+ serverProps.put("hbase.coprocessor.region.classes", FailingRegionObserver.class.getName());
+ serverProps.put("hbase.coprocessor.abortonerror", "false");
+ serverProps.put(Indexer.CHECK_VERSION_CONF_KEY, "false");
+ Map<String, String> clientProps = Collections.singletonMap(QueryServices.TRANSACTIONS_ENABLED, "true");
+ setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator()));
+ createTablesWithABitOfData();
}
@Parameters(name="transactional = {0}")
http://git-wip-us.apache.org/repos/asf/phoenix/blob/0701e06d/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixClientRpcIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixClientRpcIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixClientRpcIT.java
index 0c61b55..2ccf490 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixClientRpcIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixClientRpcIT.java
@@ -50,6 +50,7 @@ public class PhoenixClientRpcIT extends BaseOwnClusterHBaseManagedTimeIT {
@AfterClass
public static void cleanUpAfterTestSuite() throws Exception {
TestPhoenixIndexRpcSchedulerFactory.reset();
+ tearDownMiniCluster();
}
@Test
http://git-wip-us.apache.org/repos/asf/phoenix/blob/0701e06d/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index fa5df31..0c1c71b 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -121,16 +121,20 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.IntegrationTestingUtility;
import org.apache.hadoop.hbase.TableNotEnabledException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory;
import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
import org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory;
+import org.apache.hadoop.hbase.master.LoadBalancer;
import org.apache.hadoop.hbase.regionserver.LocalIndexMerger;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.end2end.BaseClientManagedTimeIT;
import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.hbase.index.balancer.IndexLoadBalancer;
+import org.apache.phoenix.hbase.index.master.IndexMasterObserver;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver;
import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
@@ -698,6 +702,9 @@ public abstract class BaseTest {
conf.setInt(HConstants.REGION_SERVER_HANDLER_COUNT, 5);
conf.setInt("hbase.regionserver.metahandler.count", 2);
conf.setInt(HConstants.MASTER_HANDLER_COUNT, 2);
+ conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName());
+ conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, IndexLoadBalancer.class,
+ LoadBalancer.class);
conf.setClass("hbase.coprocessor.regionserver.classes", LocalIndexMerger.class,
RegionServerObserver.class);
conf.setInt("dfs.namenode.handler.count", 2);
[52/52] [abbrv] phoenix git commit: Merge from master;
Temporary fix for CALCITE-1018
Posted by ma...@apache.org.
Merge from master; Temporary fix for CALCITE-1018
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/5a833c32
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/5a833c32
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/5a833c32
Branch: refs/heads/calcite
Commit: 5a833c326ff9d542792a3c261884b78cadfe4c61
Parents: 271de4d f244feb
Author: maryannxue <we...@intel.com>
Authored: Thu Dec 10 21:42:56 2015 -0500
Committer: maryannxue <we...@intel.com>
Committed: Thu Dec 10 21:42:56 2015 -0500
----------------------------------------------------------------------
bin/performance.py | 10 +-
bin/pherf-cluster.py | 83 +
bin/pherf-standalone.py | 71 +
bin/phoenix_sandbox.py | 7 +-
bin/phoenix_utils.py | 56 +-
bin/psql.py | 36 +-
bin/queryserver.py | 4 +-
bin/sqlline-thin.py | 74 +-
bin/sqlline.py | 42 +-
bin/tephra | 310 +
bin/tephra-env.sh | 55 +
bin/traceserver.py | 188 +
dev/make_rc.sh | 7 +-
phoenix-assembly/pom.xml | 4 +-
phoenix-assembly/src/build/client.xml | 4 +
.../src/build/components/all-common-files.xml | 10 +
.../src/build/server-without-antlr.xml | 44 -
phoenix-assembly/src/build/server.xml | 14 +-
phoenix-core/pom.xml | 32 +-
.../apache/phoenix/end2end/AlterTableIT.java | 211 +-
.../phoenix/end2end/AlterTableWithViewsIT.java | 56 +-
.../org/apache/phoenix/end2end/ArrayIT.java | 53 +
.../phoenix/end2end/CSVCommonsLoaderIT.java | 40 +-
.../ConnectionQueryServicesTestImpl.java | 5 +-
.../phoenix/end2end/ConnectionUtilIT.java | 84 +
.../org/apache/phoenix/end2end/DateTimeIT.java | 34 +
.../org/apache/phoenix/end2end/DeleteIT.java | 63 +
.../apache/phoenix/end2end/DerivedTableIT.java | 27 +
.../org/apache/phoenix/end2end/GroupByIT.java | 72 +
.../org/apache/phoenix/end2end/HashJoinIT.java | 545 +-
.../apache/phoenix/end2end/HashJoinMoreIT.java | 678 +
.../phoenix/end2end/ParallelIteratorsIT.java | 29 +
.../org/apache/phoenix/end2end/QueryMoreIT.java | 33 +
.../phoenix/end2end/RowValueConstructorIT.java | 61 +
.../org/apache/phoenix/end2end/SequenceIT.java | 24 +
.../apache/phoenix/end2end/SkipScanQueryIT.java | 2 +-
.../apache/phoenix/end2end/SortMergeJoinIT.java | 297 +-
.../phoenix/end2end/SortMergeJoinMoreIT.java | 314 +
.../phoenix/end2end/StatsCollectorIT.java | 33 +-
.../StatsCollectorWithSplitsAndMultiCFIT.java | 10 +-
.../end2end/SubqueryUsingSortMergeJoinIT.java | 2 -
.../org/apache/phoenix/end2end/UnionAllIT.java | 34 +
.../apache/phoenix/end2end/UpsertSelectIT.java | 586 +
.../apache/phoenix/end2end/UpsertValuesIT.java | 284 +
.../end2end/index/BaseMutableIndexIT.java | 1170 -
.../end2end/index/GlobalMutableIndexIT.java | 26 -
.../phoenix/end2end/index/ImmutableIndexIT.java | 482 +-
.../end2end/index/IndexExpressionIT.java | 76 +-
.../apache/phoenix/end2end/index/IndexIT.java | 916 +
.../phoenix/end2end/index/IndexMetadataIT.java | 44 +-
.../phoenix/end2end/index/LocalIndexIT.java | 6 +-
.../end2end/index/LocalMutableIndexIT.java | 26 -
.../end2end/index/MutableIndexFailureIT.java | 556 +-
.../phoenix/end2end/index/MutableIndexIT.java | 597 +
.../phoenix/end2end/index/SaltedIndexIT.java | 2 +-
.../end2end/index/txn/MutableRollbackIT.java | 511 +
.../phoenix/end2end/index/txn/RollbackIT.java | 168 +
.../end2end/index/txn/TxWriteFailureIT.java | 192 +
.../apache/phoenix/execute/PartialCommitIT.java | 178 +-
.../EndToEndCoveredColumnsIndexBuilderIT.java | 18 +-
.../example/EndToEndCoveredIndexingIT.java | 1633 +-
.../EndtoEndIndexingWithCompressionIT.java | 31 +-
.../covered/example/FailWithoutRetriesIT.java | 196 +-
.../phoenix/iterate/PhoenixQueryTimeoutIT.java | 114 +
.../phoenix/mapreduce/CsvBulkLoadToolIT.java | 79 +-
.../apache/phoenix/mapreduce/IndexToolIT.java | 18 +-
.../apache/phoenix/rpc/PhoenixClientRpcIT.java | 1 +
.../org/apache/phoenix/rpc/UpdateCacheIT.java | 111 +-
.../phoenix/rpc/UpdateCacheWithScnIT.java | 41 +
.../org/apache/phoenix/tx/TransactionIT.java | 551 +
.../org/apache/phoenix/tx/TxCheckpointIT.java | 379 +
phoenix-core/src/main/antlr3/PhoenixSQL.g | 27 +-
.../hbase/regionserver/LocalIndexSplitter.java | 12 +-
.../regionserver/wal/IndexedWALEditCodec.java | 56 +-
.../phoenix/cache/IndexMetaDataCache.java | 22 +
.../apache/phoenix/cache/ServerCacheClient.java | 18 +-
.../org/apache/phoenix/cache/TenantCache.java | 2 +-
.../apache/phoenix/cache/TenantCacheImpl.java | 13 +-
.../apache/phoenix/calcite/PhoenixSchema.java | 2 +-
.../metadata/PhoenixRelMdMaxRowCount.java | 31 +
.../metadata/PhoenixRelMetadataProvider.java | 1 +
.../calcite/rel/PhoenixRelImplementorImpl.java | 4 +-
.../phoenix/compile/BaseMutationPlan.java | 67 +
.../phoenix/compile/CreateFunctionCompiler.java | 17 +-
.../phoenix/compile/CreateIndexCompiler.java | 28 +-
.../phoenix/compile/CreateSequenceCompiler.java | 32 +-
.../phoenix/compile/CreateTableCompiler.java | 29 +-
.../apache/phoenix/compile/DeleteCompiler.java | 150 +-
.../phoenix/compile/DropSequenceCompiler.java | 24 +-
.../apache/phoenix/compile/FromCompiler.java | 19 +-
.../apache/phoenix/compile/JoinCompiler.java | 2 +-
.../phoenix/compile/ListJarsQueryPlan.java | 14 +-
.../apache/phoenix/compile/MutationPlan.java | 6 +-
.../apache/phoenix/compile/OrderByCompiler.java | 10 +-
.../apache/phoenix/compile/PostDDLCompiler.java | 89 +-
.../phoenix/compile/PostIndexDDLCompiler.java | 5 +-
.../phoenix/compile/ProjectionCompiler.java | 16 +-
.../apache/phoenix/compile/QueryCompiler.java | 33 +-
.../org/apache/phoenix/compile/ScanRanges.java | 110 +-
.../phoenix/compile/StatementContext.java | 14 +-
.../apache/phoenix/compile/StatementPlan.java | 8 +-
.../phoenix/compile/SubselectRewriter.java | 31 +-
.../apache/phoenix/compile/TraceQueryPlan.java | 29 +-
.../compile/TupleProjectionCompiler.java | 4 +-
.../apache/phoenix/compile/UnionCompiler.java | 4 +-
.../apache/phoenix/compile/UpsertCompiler.java | 206 +-
.../apache/phoenix/compile/WhereOptimizer.java | 9 +-
.../coprocessor/BaseScannerRegionObserver.java | 10 +-
.../coprocessor/MetaDataEndpointImpl.java | 81 +-
.../phoenix/coprocessor/MetaDataProtocol.java | 10 +-
.../phoenix/coprocessor/ScanRegionObserver.java | 12 +-
.../coprocessor/ServerCachingEndpointImpl.java | 8 +-
.../coprocessor/ServerCachingProtocol.java | 5 +-
.../UngroupedAggregateRegionObserver.java | 44 +-
.../coprocessor/generated/PTableProtos.java | 235 +-
.../generated/ServerCachingProtos.java | 120 +-
.../phoenix/exception/SQLExceptionCode.java | 108 +-
.../apache/phoenix/execute/AggregatePlan.java | 10 +-
.../apache/phoenix/execute/BaseQueryPlan.java | 70 +-
.../apache/phoenix/execute/DelegateHTable.java | 300 +
.../phoenix/execute/DelegateQueryPlan.java | 12 +
.../apache/phoenix/execute/HashJoinPlan.java | 27 +-
.../apache/phoenix/execute/MutationState.java | 1009 +-
.../org/apache/phoenix/execute/ScanPlan.java | 13 +-
.../phoenix/execute/SortMergeJoinPlan.java | 17 +-
.../org/apache/phoenix/execute/UnionPlan.java | 20 +-
.../phoenix/expression/CoerceExpression.java | 8 +-
.../expression/RowKeyColumnExpression.java | 2 +-
.../org/apache/phoenix/hbase/index/Indexer.java | 53 -
.../phoenix/hbase/index/MultiMutation.java | 86 +
.../apache/phoenix/hbase/index/ValueGetter.java | 4 +-
.../hbase/index/builder/BaseIndexBuilder.java | 163 +-
.../hbase/index/builder/BaseIndexCodec.java | 47 +
.../hbase/index/builder/IndexBuildManager.java | 32 +-
.../hbase/index/builder/IndexBuilder.java | 75 +-
.../covered/CoveredColumnsIndexBuilder.java | 491 -
.../phoenix/hbase/index/covered/IndexCodec.java | 163 +-
.../hbase/index/covered/IndexMetaData.java | 22 +
.../hbase/index/covered/LocalTableState.java | 447 +-
.../hbase/index/covered/NonTxIndexBuilder.java | 404 +
.../phoenix/hbase/index/covered/TableState.java | 40 +-
.../index/covered/data/LazyValueGetter.java | 13 +-
.../example/CoveredColumnIndexCodec.java | 604 +-
.../CoveredColumnIndexSpecifierBuilder.java | 4 +-
.../covered/example/CoveredColumnIndexer.java | 15 +-
.../index/covered/update/ColumnTracker.java | 3 +-
.../covered/update/IndexUpdateManager.java | 5 +-
.../hbase/index/scanner/EmptyScanner.java | 8 +-
.../phoenix/hbase/index/scanner/Scanner.java | 2 +-
.../hbase/index/scanner/ScannerBuilder.java | 11 +-
.../hbase/index/write/IndexFailurePolicy.java | 4 +-
.../apache/phoenix/index/BaseIndexCodec.java | 59 -
.../apache/phoenix/index/IndexMaintainer.java | 116 +-
.../phoenix/index/IndexMetaDataCacheClient.java | 12 +-
.../index/IndexMetaDataCacheFactory.java | 19 +-
.../phoenix/index/PhoenixIndexBuilder.java | 49 +-
.../apache/phoenix/index/PhoenixIndexCodec.java | 207 +-
.../index/PhoenixIndexFailurePolicy.java | 2 +-
.../phoenix/index/PhoenixIndexMetaData.java | 103 +
.../index/PhoenixTransactionalIndexer.java | 505 +
.../phoenix/iterate/BaseResultIterators.java | 28 +-
.../phoenix/iterate/ChunkedResultIterator.java | 25 +-
.../apache/phoenix/iterate/ExplainTable.java | 5 +
.../MaterializedComparableResultIterator.java | 72 +
.../iterate/MergeSortResultIterator.java | 99 +-
.../phoenix/iterate/ParallelIterators.java | 2 +-
.../apache/phoenix/iterate/SerialIterators.java | 2 +-
.../phoenix/iterate/TableResultIterator.java | 28 +-
.../phoenix/iterate/UnionResultIterators.java | 11 +-
.../apache/phoenix/jdbc/PhoenixConnection.java | 88 +-
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 7 +
.../org/apache/phoenix/jdbc/PhoenixDriver.java | 84 +-
.../phoenix/jdbc/PhoenixEmbeddedDriver.java | 4 +
.../phoenix/jdbc/PhoenixPreparedStatement.java | 1 -
.../apache/phoenix/jdbc/PhoenixResultSet.java | 7 +
.../apache/phoenix/jdbc/PhoenixStatement.java | 276 +-
.../apache/phoenix/join/HashCacheClient.java | 3 +-
.../apache/phoenix/join/HashCacheFactory.java | 2 +-
.../phoenix/mapreduce/AbstractBulkLoadTool.java | 402 +
.../phoenix/mapreduce/CsvBulkImportUtil.java | 40 +-
.../phoenix/mapreduce/CsvBulkLoadTool.java | 463 +-
.../phoenix/mapreduce/CsvToKeyValueMapper.java | 222 +-
.../mapreduce/FormatToKeyValueMapper.java | 264 +
.../mapreduce/FormatToKeyValueReducer.java | 54 +
.../ImportPreUpsertKeyValueProcessor.java | 3 +-
.../phoenix/mapreduce/JsonBulkLoadTool.java | 53 +
.../phoenix/mapreduce/JsonToKeyValueMapper.java | 75 +
.../mapreduce/MultiHfileOutputFormat.java | 716 +
.../phoenix/mapreduce/PhoenixRecordReader.java | 2 +-
.../mapreduce/bulkload/TableRowkeyPair.java | 134 +
.../mapreduce/bulkload/TargetTableRef.java | 70 +
.../bulkload/TargetTableRefFunctions.java | 95 +
.../mapreduce/index/DirectHTableWriter.java | 10 +-
.../index/PhoenixIndexImportDirectMapper.java | 92 +-
.../phoenix/mapreduce/util/ConnectionUtil.java | 17 +-
.../util/PhoenixConfigurationUtil.java | 29 +-
.../apache/phoenix/optimize/QueryOptimizer.java | 15 +-
.../org/apache/phoenix/parse/ColumnDef.java | 227 +-
.../phoenix/parse/ColumnDefInPkConstraint.java | 44 +
.../apache/phoenix/parse/ParseNodeFactory.java | 21 +-
.../phoenix/parse/PrimaryKeyConstraint.java | 48 +-
.../phoenix/query/ConnectionQueryServices.java | 6 +-
.../query/ConnectionQueryServicesImpl.java | 695 +-
.../query/ConnectionlessQueryServicesImpl.java | 60 +-
.../query/DelegateConnectionQueryServices.java | 27 +-
.../java/org/apache/phoenix/query/KeyRange.java | 34 +-
.../apache/phoenix/query/MetaDataMutated.java | 7 +-
.../apache/phoenix/query/QueryConstants.java | 24 +-
.../org/apache/phoenix/query/QueryServices.java | 18 +
.../phoenix/query/QueryServicesOptions.java | 33 +-
.../apache/phoenix/schema/DelegateColumn.java | 15 +-
.../apache/phoenix/schema/DelegateTable.java | 18 +-
.../apache/phoenix/schema/MetaDataClient.java | 698 +-
.../java/org/apache/phoenix/schema/PColumn.java | 5 +
.../org/apache/phoenix/schema/PColumnImpl.java | 19 +-
.../java/org/apache/phoenix/schema/PDatum.java | 1 +
.../org/apache/phoenix/schema/PMetaData.java | 4 +-
.../apache/phoenix/schema/PMetaDataImpl.java | 109 +-
.../java/org/apache/phoenix/schema/PTable.java | 9 +-
.../org/apache/phoenix/schema/PTableImpl.java | 108 +-
.../org/apache/phoenix/schema/PTableRef.java | 65 +
.../org/apache/phoenix/schema/SaltingUtil.java | 2 +-
.../apache/phoenix/schema/TableProperty.java | 5 +-
.../org/apache/phoenix/schema/TableRef.java | 12 +-
.../phoenix/schema/tuple/ValueGetterTuple.java | 16 +-
.../apache/phoenix/schema/types/PBoolean.java | 210 +-
.../org/apache/phoenix/schema/types/PChar.java | 6 +-
.../schema/types/PUnsignedDoubleArray.java | 4 +-
.../schema/types/PUnsignedFloatArray.java | 4 +-
.../phoenix/schema/types/PUnsignedIntArray.java | 4 +-
.../schema/types/PUnsignedLongArray.java | 4 +-
.../schema/types/PUnsignedSmallintArray.java | 4 +-
.../schema/types/PUnsignedTinyintArray.java | 4 +-
.../apache/phoenix/schema/types/PVarchar.java | 6 +-
.../phoenix/trace/PhoenixMetricsSink.java | 9 +-
.../apache/phoenix/util/CSVCommonsLoader.java | 173 +-
.../org/apache/phoenix/util/ColumnInfo.java | 89 +-
.../java/org/apache/phoenix/util/IndexUtil.java | 29 +-
.../apache/phoenix/util/InstanceResolver.java | 26 +-
.../org/apache/phoenix/util/MetaDataUtil.java | 11 +-
.../phoenix/util/PhoenixContextExecutor.java | 9 +-
.../org/apache/phoenix/util/PhoenixRuntime.java | 18 +-
.../java/org/apache/phoenix/util/QueryUtil.java | 22 +-
.../java/org/apache/phoenix/util/ScanUtil.java | 43 +-
.../org/apache/phoenix/util/SchemaUtil.java | 174 +-
.../org/apache/phoenix/util/ServerUtil.java | 20 +-
.../apache/phoenix/util/TransactionUtil.java | 105 +
.../org/apache/phoenix/util/UpsertExecutor.java | 156 +
.../phoenix/util/csv/CsvUpsertExecutor.java | 133 +-
.../phoenix/util/json/JsonUpsertExecutor.java | 209 +
.../util/json/ObjectToArrayConverter.java | 69 +
.../phoenix/calcite/ToExpressionTest.java | 2 +-
.../phoenix/compile/QueryCompilerTest.java | 91 +-
.../phoenix/compile/QueryOptimizerTest.java | 17 +
.../phoenix/compile/ViewCompilerTest.java | 29 +-
.../phoenix/compile/WhereOptimizerTest.java | 2 +
.../phoenix/execute/CorrelatePlanTest.java | 4 +-
.../phoenix/execute/UnnestArrayPlanTest.java | 4 +-
.../expression/ColumnExpressionTest.java | 8 +-
.../phoenix/filter/SkipScanBigFilterTest.java | 5 +-
.../covered/CoveredIndexCodecForTesting.java | 70 +-
.../example/TestCoveredColumnIndexCodec.java | 33 +-
.../phoenix/index/IndexMaintainerTest.java | 6 +-
.../iterate/AggregateResultScannerTest.java | 4 +
.../iterate/MergeSortResultIteratorTest.java | 159 +-
.../apache/phoenix/jdbc/PhoenixDriverTest.java | 16 +
.../jdbc/PhoenixPreparedStatementTest.java | 94 +
.../apache/phoenix/jdbc/PhoenixTestDriver.java | 11 +-
.../phoenix/mapreduce/BulkLoadToolTest.java | 78 +
.../mapreduce/CsvBulkImportUtilTest.java | 67 +-
.../phoenix/mapreduce/CsvBulkLoadToolTest.java | 69 -
.../mapreduce/CsvToKeyValueMapperTest.java | 84 +-
.../mapreduce/FormatToKeyValueMapperTest.java | 102 +
.../util/PhoenixConfigurationUtilTest.java | 73 +
.../apache/phoenix/parse/QueryParserTest.java | 99 +-
.../query/BaseConnectionlessQueryTest.java | 2 +-
.../java/org/apache/phoenix/query/BaseTest.java | 250 +-
.../phoenix/query/ConnectionlessTest.java | 5 +-
.../query/ParallelIteratorsSplitTest.java | 22 +-
.../phoenix/query/QueryServicesTestImpl.java | 6 +-
.../org/apache/phoenix/schema/MutationTest.java | 74 +
.../phoenix/schema/PMetaDataImplTest.java | 76 +-
.../apache/phoenix/schema/RowKeySchemaTest.java | 2 +-
.../phoenix/schema/RowKeyValueAccessorTest.java | 2 +-
.../schema/types/PDataTypeForArraysTest.java | 11 +
.../phoenix/schema/types/PDataTypeTest.java | 33 +
.../util/AbstractUpsertExecutorTest.java | 136 +
.../org/apache/phoenix/util/ColumnInfoTest.java | 27 +
.../phoenix/util/GeneratePerformanceData.java | 20 +-
.../java/org/apache/phoenix/util/TestUtil.java | 39 +-
.../phoenix/util/csv/CsvUpsertExecutorTest.java | 143 +-
.../util/json/JsonUpsertExecutorTest.java | 53 +
phoenix-flume/pom.xml | 9 +-
.../flume/serializer/CustomSerializer.java | 9 +-
phoenix-pherf/README.md | 2 +-
phoenix-pherf/cluster/pherf.sh | 33 -
phoenix-pherf/config/pherf.properties | 5 +
phoenix-pherf/pom.xml | 39 +-
.../org/apache/phoenix/pherf/DataIngestIT.java | 44 +-
.../apache/phoenix/pherf/ResultBaseTestIT.java | 11 +-
phoenix-pherf/src/main/assembly/standalone.xml | 51 -
.../java/org/apache/phoenix/pherf/Pherf.java | 60 +-
.../apache/phoenix/pherf/PherfConstants.java | 50 +-
.../phoenix/pherf/configuration/Scenario.java | 13 +
.../phoenix/pherf/jmx/MonitorManager.java | 14 +-
.../phoenix/pherf/result/QueryResult.java | 55 +-
.../phoenix/pherf/result/ResultHandler.java | 6 +
.../phoenix/pherf/result/ResultManager.java | 98 +-
.../apache/phoenix/pherf/result/ResultUtil.java | 58 +-
.../phoenix/pherf/result/file/Header.java | 2 +-
.../pherf/result/file/ResultFileDetails.java | 5 +
.../pherf/result/impl/CSVFileResultHandler.java | 89 +
.../pherf/result/impl/CSVResultHandler.java | 108 +-
.../pherf/result/impl/DefaultResultHandler.java | 67 +
.../pherf/result/impl/ImageResultHandler.java | 50 +-
.../pherf/result/impl/XMLResultHandler.java | 44 +-
.../phoenix/pherf/rules/RulesApplier.java | 70 +-
.../pherf/util/GoogleChartGenerator.java | 394 +
.../apache/phoenix/pherf/util/PhoenixUtil.java | 265 +-
.../pherf/workload/MultiThreadedRunner.java | 11 +-
.../pherf/workload/MultithreadedDiffer.java | 4 +-
.../phoenix/pherf/workload/QueryExecutor.java | 56 +-
.../phoenix/pherf/workload/QueryVerifier.java | 30 -
.../pherf/workload/WorkloadExecutor.java | 17 +-
.../phoenix/pherf/workload/WriteWorkload.java | 23 +-
.../org/apache/phoenix/pherf/PherfTest.java | 2 +-
.../apache/phoenix/pherf/ResultBaseTest.java | 17 +-
.../org/apache/phoenix/pherf/ResultTest.java | 101 +-
.../apache/phoenix/pherf/RuleGeneratorTest.java | 51 +-
.../test/resources/datamodel/test_schema.sql | 1 +
.../resources/datamodel/test_schema_mt_view.sql | 1 -
.../test/resources/scenario/test_scenario.xml | 39 +-
phoenix-pherf/standalone/pherf.sh | 28 -
phoenix-pig/pom.xml | 9 +-
.../java/org/apache/phoenix/pig/BasePigIT.java | 83 +
.../phoenix/pig/PhoenixHBaseLoaderIT.java | 423 +-
.../phoenix/pig/PhoenixHBaseStorerIT.java | 43 +-
.../phoenix/pig/udf/ReserveNSequenceTestIT.java | 128 +-
.../phoenix/pig/udf/ReserveNSequence.java | 88 +-
phoenix-protocol/src/main/PTable.proto | 2 +
.../src/main/ServerCachingService.proto | 1 +
phoenix-server-client/pom.xml | 2 +-
.../queryserver/client/ThinClientUtil.java | 11 +-
phoenix-server/pom.xml | 10 +-
.../src/build/query-server-runnable.xml | 8 +-
.../phoenix/end2end/QueryServerBasicsIT.java | 4 +-
.../apache/phoenix/queryserver/server/Main.java | 89 +-
phoenix-spark/pom.xml | 11 +-
phoenix-spark/src/it/resources/setup.sql | 13 +-
.../apache/phoenix/spark/PhoenixSparkIT.scala | 195 +-
.../phoenix/spark/DataFrameFunctions.scala | 15 +-
.../org/apache/phoenix/spark/PhoenixRDD.scala | 28 +-
.../phoenix/spark/PhoenixRecordWritable.scala | 45 +-
.../apache/phoenix/spark/PhoenixRelation.scala | 25 +-
phoenix-tracing-webapp/README.md | 15 +
phoenix-tracing-webapp/pom.xml | 165 +
.../src/build/trace-server-runnable.xml | 62 +
.../src/main/config/checkstyle/checker.xml | 281 +
.../src/main/config/checkstyle/header.txt | 16 +
.../src/main/config/checkstyle/suppressions.xml | 46 +
.../tracingwebapp/http/ConnectionFactory.java | 43 +
.../tracingwebapp/http/EntityFactory.java | 101 +
.../apache/phoenix/tracingwebapp/http/Main.java | 81 +
.../tracingwebapp/http/TraceServlet.java | 152 +
.../src/main/webapp/WEB-INF/web.xml | 15 +
.../src/main/webapp/css/bootstrap-theme.css | 476 +
.../src/main/webapp/css/bootstrap-theme.css.map | 1 +
.../src/main/webapp/css/bootstrap.css | 6584 +++++
.../src/main/webapp/css/bootstrap.css.map | 1 +
.../src/main/webapp/css/font-awesome.css | 1801 ++
.../src/main/webapp/fonts/FontAwesome.otf | Bin 0 -> 93888 bytes
.../main/webapp/fonts/fontawesome-webfont.eot | Bin 0 -> 60767 bytes
.../main/webapp/fonts/fontawesome-webfont.svg | 565 +
.../main/webapp/fonts/fontawesome-webfont.ttf | Bin 0 -> 122092 bytes
.../main/webapp/fonts/fontawesome-webfont.woff | Bin 0 -> 71508 bytes
.../main/webapp/fonts/fontawesome-webfont.woff2 | Bin 0 -> 56780 bytes
.../fonts/glyphicons-halflings-regular.eot | Bin 0 -> 20127 bytes
.../fonts/glyphicons-halflings-regular.svg | 288 +
.../fonts/glyphicons-halflings-regular.ttf | Bin 0 -> 45404 bytes
.../fonts/glyphicons-halflings-regular.woff | Bin 0 -> 23424 bytes
.../fonts/glyphicons-halflings-regular.woff2 | Bin 0 -> 18028 bytes
.../src/main/webapp/index.html | 110 +
.../src/main/webapp/js/api/chart-model.js | 177 +
.../src/main/webapp/js/app.js | 52 +
.../src/main/webapp/js/config/chart-config.js | 45 +
.../js/controllers/accordion-controllers.js | 29 +
.../controllers/dependency-tree-controllers.js | 143 +
.../webapp/js/controllers/list-controllers.js | 22 +
.../webapp/js/controllers/search-controllers.js | 71 +
.../js/controllers/timeline-controllers.js | 153 +
.../js/controllers/trace-count-controllers.js | 40 +
.../trace-distribution-controllers.js | 39 +
.../js/factories/statement-factory-config.js | 14 +
.../webapp/js/factories/statement-factory.js | 29 +
.../src/main/webapp/js/lib/angular-mocks.js | 2468 ++
.../src/main/webapp/js/lib/angular-route.js | 995 +
.../src/main/webapp/js/lib/angular.js | 26309 +++++++++++++++++
.../src/main/webapp/js/lib/bootstrap.js | 2317 ++
.../src/main/webapp/js/lib/jquery-2.1.4.js | 9210 ++++++
.../src/main/webapp/js/lib/jquery.min.js | 5 +
.../src/main/webapp/js/lib/ng-google-chart.js | 261 +
.../src/main/webapp/js/lib/ui-bootstrap-tpls.js | 4840 +++
.../js/services/generate-statement-service.js | 46 +
.../src/main/webapp/partials/about.html | 10 +
.../src/main/webapp/partials/chart.html | 24 +
.../src/main/webapp/partials/contact.html | 5 +
.../main/webapp/partials/dependency-tree.html | 11 +
.../src/main/webapp/partials/google-chart.html | 23 +
.../src/main/webapp/partials/help.html | 36 +
.../src/main/webapp/partials/home.html | 20 +
.../src/main/webapp/partials/list.html | 39 +
.../src/main/webapp/partials/phoenix-trace.html | 60 +
.../src/main/webapp/partials/search.html | 58 +
.../src/test/webapp/js/specs/app-route-spec.js | 55 +
.../test/webapp/js/specs/timeline-ctrl-spec.js | 27 +
.../webapp/js/specs/trace-list-ctrl-spec.js | 89 +
.../webapp/js/specs/tracing-app-ctrl-spec.js | 23 +
pom.xml | 14 +-
418 files changed, 80900 insertions(+), 10109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --cc phoenix-core/pom.xml
index b1a0883,8fe7ecc..bf2cc7c
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@@ -272,34 -202,11 +272,32 @@@
<artifactId>maven-failsafe-plugin</artifactId>
</plugin>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>${maven-dependency-plugin.version}</version>
- </plugin>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
+ <!-- Extract parser grammar template from optiq-core.jar and put
+ it under ${project.build.directory} where all freemarker templates are. -->
+ <id>unpack-parser-template</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-core</artifactId>
+ <type>jar</type>
+ <overWrite>true</overWrite>
+ <outputDirectory>${project.build.directory}/</outputDirectory>
+ <includes>**/Parser.jj</includes>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ <execution>
<!-- generates the file that will be used by the sandbox script in the dev env -->
<id>create-phoenix-generated-classpath</id>
<goals>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixSchema.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixSchema.java
index 65e7bb8,0000000..bd98371
mode 100644,000000..100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixSchema.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixSchema.java
@@@ -1,346 -1,0 +1,346 @@@
+package org.apache.phoenix.calcite;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.materialize.MaterializationService;
+import org.apache.calcite.schema.*;
+import org.apache.calcite.schema.impl.ViewTable;
+import org.apache.phoenix.compile.ColumnResolver;
+import org.apache.phoenix.compile.FromCompiler;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.parse.ColumnDef;
+import org.apache.phoenix.parse.NamedTableNode;
+import org.apache.phoenix.parse.TableName;
+import org.apache.phoenix.schema.MetaDataClient;
+import org.apache.phoenix.schema.PColumn;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.LinkType;
+import org.apache.phoenix.schema.PTable.ViewType;
+import org.apache.phoenix.schema.PTableImpl;
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.schema.TableNotFoundException;
+import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.util.IndexUtil;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+/**
+ * Implementation of Calcite's {@link Schema} SPI for Phoenix.
+ *
+ * TODO
+ * 1) change this to non-caching mode??
+ * 2) how to deal with define indexes and views since they require a CalciteSchema
+ * instance??
+ *
+ */
+public class PhoenixSchema implements Schema {
+ public static final Factory FACTORY = new Factory();
+
+ public final PhoenixConnection pc;
+
+ protected final String name;
+ protected final String schemaName;
+ protected final MetaDataClient client;
+
+ protected final Set<String> subSchemaNames;
+ protected final Map<String, PTable> tableMap;
+ protected final Map<String, ViewDef> viewDefMap;
+ protected final Map<String, Function> functionMap;
+ protected final Map<String, PhoenixSequence> sequenceMap;
+
+ private PhoenixSchema(String name, String schemaName, PhoenixConnection pc) {
+ this.name = name;
+ this.schemaName = schemaName;
+ this.pc = pc;
+ this.client = new MetaDataClient(pc);
+ this.tableMap = Maps.<String, PTable> newHashMap();
+ this.viewDefMap = Maps.<String, ViewDef> newHashMap();
+ this.functionMap = Maps.<String, Function> newHashMap();
+ this.sequenceMap = Maps.<String, PhoenixSequence> newHashMap();
+ loadTables();
+ loadSequences();
+ this.subSchemaNames = schemaName == null ?
+ ImmutableSet.<String> copyOf(loadSubSchemaNames())
+ : Collections.<String> emptySet();
+ }
+
+ private Set<String> loadSubSchemaNames() {
+ try {
+ DatabaseMetaData md = pc.getMetaData();
+ ResultSet rs = md.getSchemas();
+ Set<String> subSchemaNames = Sets.newHashSet();
+ while (rs.next()) {
+ String schemaName = rs.getString(PhoenixDatabaseMetaData.TABLE_SCHEM);
+ if (schemaName != null) {
+ subSchemaNames.add(schemaName);
+ }
+ }
+ // TODO FIXME: Remove this after PHOENIX-2489.
+ String tenantId = pc.getTenantId() == null ? null : pc.getTenantId().getString();
+ String q = "select " + PhoenixDatabaseMetaData.SEQUENCE_SCHEMA
+ + " from " + PhoenixDatabaseMetaData.SEQUENCE_FULLNAME_ESCAPED
+ + " where " + PhoenixDatabaseMetaData.SEQUENCE_SCHEMA
+ + " is not null"
+ + " and " + PhoenixDatabaseMetaData.TENANT_ID
+ + (tenantId == null ? " is null" : " = '" + tenantId + "'");
+ rs = pc.createStatement().executeQuery(q);
+ while (rs.next()) {
+ String schemaName = rs.getString(1);
+ subSchemaNames.add(schemaName);
+ }
+ return subSchemaNames;
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void loadTables() {
+ try {
+ DatabaseMetaData md = pc.getMetaData();
+ ResultSet rs = md.getTables(null, schemaName == null ? "" : schemaName, null, null);
+ while (rs.next()) {
+ String tableName = rs.getString(PhoenixDatabaseMetaData.TABLE_NAME);
+ String tableType = rs.getString(PhoenixDatabaseMetaData.TABLE_TYPE);
+ String viewType = rs.getString(PhoenixDatabaseMetaData.VIEW_TYPE);
+ if (!tableType.equals(PTableType.VIEW.getValue().getString())
+ || ViewType.MAPPED.name().equals(viewType)) {
+ try {
+ ColumnResolver x = FromCompiler.getResolver(
+ NamedTableNode.create(
+ null,
+ TableName.create(schemaName, tableName),
+ ImmutableList.<ColumnDef>of()), pc);
+ final List<TableRef> tables = x.getTables();
+ assert tables.size() == 1;
+ PTable pTable = tables.get(0).getTable();
+ if (pc.getTenantId() == null && pTable.isMultiTenant()) {
+ pTable = fixTableMultiTenancy(pTable);
+ }
+ tableMap.put(tableName, pTable);
+ } catch (TableNotFoundException e) {
+ // Multi-tenant table with non-tenant-specific connection.
+ }
+ } else {
+ boolean isMultiTenant = rs.getBoolean(PhoenixDatabaseMetaData.MULTI_TENANT);
+ if (pc.getTenantId() != null || !isMultiTenant) {
+ String viewSql = rs.getString(PhoenixDatabaseMetaData.VIEW_STATEMENT);
+ if (viewSql == null) {
+ String q = "select " + PhoenixDatabaseMetaData.COLUMN_FAMILY
+ + " from " + PhoenixDatabaseMetaData.SYSTEM_CATALOG
+ + " where " + PhoenixDatabaseMetaData.TABLE_SCHEM
+ + (schemaName == null ? " is null" : " = '" + schemaName + "'")
+ + " and " + PhoenixDatabaseMetaData.TABLE_NAME
+ + " = '" + tableName + "'"
+ + " and " + PhoenixDatabaseMetaData.COLUMN_FAMILY
+ + " is not null"
+ + " and " + PhoenixDatabaseMetaData.LINK_TYPE
+ + " = " + LinkType.PHYSICAL_TABLE.getSerializedValue();
+ ResultSet rs2 = pc.createStatement().executeQuery(q);
+ if (!rs2.next()) {
+ throw new SQLException("View link not found for " + tableName);
+ }
+ String parentTableName = rs2.getString(PhoenixDatabaseMetaData.COLUMN_FAMILY);
+ viewSql = "select * from " + parentTableName;
+ }
+ viewDefMap.put(tableName, new ViewDef(viewSql, viewType.equals(ViewType.UPDATABLE.name())));
+ }
+ }
+ }
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private PTable fixTableMultiTenancy(PTable table) throws SQLException {
+ return PTableImpl.makePTable(
+ table.getTenantId(), table.getSchemaName(), table.getTableName(), table.getType(), table.getIndexState(), table.getTimeStamp(),
+ table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), PTableImpl.getColumnsToClone(table), table.getParentSchemaName(), table.getParentTableName(),
+ table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
+ table.isWALDisabled(), false, table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
- table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable());
++ table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional());
+ }
+
+ private void loadSequences() {
+ try {
+ // TODO FIXME: Do this in loadTables() after PHOENIX-2489.
+ String tenantId = pc.getTenantId() == null ? null : pc.getTenantId().getString();
+ String q = "select " + PhoenixDatabaseMetaData.SEQUENCE_NAME
+ + " from " + PhoenixDatabaseMetaData.SEQUENCE_FULLNAME_ESCAPED
+ + " where " + PhoenixDatabaseMetaData.SEQUENCE_SCHEMA
+ + (schemaName == null ? " is null" : " = '" + schemaName + "'")
+ + " and " + PhoenixDatabaseMetaData.TENANT_ID
+ + (tenantId == null ? " is null" : " = '" + tenantId + "'");
+ ResultSet rs = pc.createStatement().executeQuery(q);
+ while (rs.next()) {
+ String sequenceName = rs.getString(1);
+ sequenceMap.put(sequenceName, new PhoenixSequence(schemaName, sequenceName, pc));
+ }
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static Schema create(String name, Map<String, Object> operand) {
+ String url = (String) operand.get("url");
+ final Properties properties = new Properties();
+ for (Map.Entry<String, Object> entry : operand.entrySet()) {
+ properties.setProperty(entry.getKey(), String.valueOf(entry.getValue()));
+ }
+ try {
+ Class.forName("org.apache.phoenix.jdbc.PhoenixDriver");
+ final Connection connection =
+ DriverManager.getConnection(url, properties);
+ final PhoenixConnection phoenixConnection =
+ connection.unwrap(PhoenixConnection.class);
+ return new PhoenixSchema(name, null, phoenixConnection);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Table getTable(String name) {
+ PTable table = tableMap.get(name);
+ if (table != null) {
+ return new PhoenixTable(pc, table);
+ }
+ PhoenixSequence sequence = sequenceMap.get(name);
+ return sequence;
+ }
+
+ @Override
+ public Set<String> getTableNames() {
+ return Sets.union(tableMap.keySet(), sequenceMap.keySet());
+ }
+
+ @Override
+ public Collection<Function> getFunctions(String name) {
+ Function func = functionMap.get(name);
+ return func == null ? Collections.<Function>emptyList() : ImmutableList.of(func);
+ }
+
+ @Override
+ public Set<String> getFunctionNames() {
+ return viewDefMap.keySet();
+ }
+
+ @Override
+ public Schema getSubSchema(String name) {
+ if (!subSchemaNames.contains(name))
+ return null;
+
+ return new PhoenixSchema(name, name, pc);
+ }
+
+ @Override
+ public Set<String> getSubSchemaNames() {
+ return subSchemaNames;
+ }
+
+ @Override
+ public Expression getExpression(SchemaPlus parentSchema, String name) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isMutable() {
+ return true;
+ }
+
+ @Override
+ public boolean contentsHaveChangedSince(long lastCheck, long now) {
+ return false;
+ }
+
+ public void initFunctionMap(CalciteSchema calciteSchema) {
+ for (Map.Entry<String, ViewDef> entry : viewDefMap.entrySet()) {
+ ViewDef viewDef = entry.getValue();
+ Function func = ViewTable.viewMacro(
+ calciteSchema.plus(), viewDef.viewSql,
+ calciteSchema.path(null), viewDef.updatable);
+ functionMap.put(entry.getKey(), func);
+ }
+ }
+
+ public void defineIndexesAsMaterializations(CalciteSchema calciteSchema) {
+ List<String> path = calciteSchema.path(null);
+ for (PTable table : tableMap.values()) {
+ if (table.getType() == PTableType.INDEX) {
+ addMaterialization(table, path, calciteSchema);
+ }
+ }
+ }
+
+ protected void addMaterialization(PTable index, List<String> path,
+ CalciteSchema calciteSchema) {
+ StringBuffer sb = new StringBuffer();
+ sb.append("SELECT");
+ for (PColumn column : PhoenixTable.getMappedColumns(index)) {
+ String indexColumnName = column.getName().getString();
+ String dataColumnName = IndexUtil.getDataColumnName(indexColumnName);
+ sb.append(",").append("\"").append(dataColumnName).append("\"");
+ sb.append(" ").append("\"").append(indexColumnName).append("\"");
+ }
+ sb.setCharAt(6, ' '); // replace first comma with space.
+ sb.append(" FROM ").append("\"").append(index.getParentName().getString()).append("\"");
+ MaterializationService.instance().defineMaterialization(
+ calciteSchema, null, sb.toString(), path, index.getTableName().getString(), true, true);
+ }
+
+ private static class ViewDef {
+ final String viewSql;
+ final boolean updatable;
+
+ ViewDef(String viewSql, boolean updatable) {
+ this.viewSql = viewSql;
+ this.updatable = updatable;
+ }
+ }
+
+ /** Schema factory that creates a
+ * {@link org.apache.phoenix.calcite.PhoenixSchema}.
+ * This allows you to create a Phoenix schema inside a model.json file.
+ *
+ * <pre>{@code
+ * {
+ * version: '1.0',
+ * defaultSchema: 'HR',
+ * schemas: [
+ * {
+ * name: 'HR',
+ * type: 'custom',
+ * factory: 'org.apache.phoenix.calcite.PhoenixSchema.Factory',
+ * operand: {
+ * url: "jdbc:phoenix:localhost",
+ * user: "scott",
+ * password: "tiger"
+ * }
+ * }
+ * ]
+ * }
+ * }</pre>
+ */
+ public static class Factory implements SchemaFactory {
+ public Schema create(SchemaPlus parentSchema, String name, Map<String, Object> operand) {
+ return PhoenixSchema.create(name, operand);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/calcite/metadata/PhoenixRelMdMaxRowCount.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/calcite/metadata/PhoenixRelMdMaxRowCount.java
index 0000000,0000000..af0a52d
new file mode 100644
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/calcite/metadata/PhoenixRelMdMaxRowCount.java
@@@ -1,0 -1,0 +1,31 @@@
++package org.apache.phoenix.calcite.metadata;
++
++import org.apache.calcite.plan.volcano.RelSubset;
++import org.apache.calcite.rel.RelNode;
++import org.apache.calcite.rel.core.Sort;
++import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider;
++import org.apache.calcite.rel.metadata.RelMetadataProvider;
++import org.apache.calcite.rex.RexLiteral;
++import org.apache.calcite.util.BuiltInMethod;
++
++public class PhoenixRelMdMaxRowCount {
++ public static final RelMetadataProvider SOURCE =
++ ReflectiveRelMetadataProvider.reflectiveSource(
++ BuiltInMethod.MAX_ROW_COUNT.method, new PhoenixRelMdMaxRowCount());
++
++ private PhoenixRelMdMaxRowCount() {
++ }
++
++ public Double getMaxRowCount(RelSubset rel) {
++ for (RelNode node : rel.getRels()) {
++ if (node instanceof Sort) {
++ Sort sort = (Sort) node;
++ if (sort.fetch != null) {
++ return (double) RexLiteral.intValue(sort.fetch);
++ }
++ }
++ }
++
++ return Double.POSITIVE_INFINITY;
++ }
++}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/calcite/metadata/PhoenixRelMetadataProvider.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/calcite/metadata/PhoenixRelMetadataProvider.java
index c9412c6,0000000..e198477
mode 100644,000000..100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/calcite/metadata/PhoenixRelMetadataProvider.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/calcite/metadata/PhoenixRelMetadataProvider.java
@@@ -1,16 -1,0 +1,17 @@@
+package org.apache.phoenix.calcite.metadata;
+
+import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
+import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider;
+import com.google.common.collect.ImmutableList;
+
+public class PhoenixRelMetadataProvider extends ChainedRelMetadataProvider {
+
+ public PhoenixRelMetadataProvider() {
+ super(ImmutableList.of(
+ PhoenixRelMdRowCount.SOURCE,
+ PhoenixRelMdCollation.SOURCE,
++ PhoenixRelMdMaxRowCount.SOURCE,
+ new DefaultRelMetadataProvider()));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixRelImplementorImpl.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixRelImplementorImpl.java
index 341342c,0000000..44f3031
mode 100644,000000..100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixRelImplementorImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixRelImplementorImpl.java
@@@ -1,191 -1,0 +1,191 @@@
+package org.apache.phoenix.calcite.rel;
+
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Stack;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.phoenix.calcite.CalciteUtils;
+import org.apache.phoenix.calcite.PhoenixSequence;
+import org.apache.phoenix.calcite.PhoenixTable;
+import org.apache.phoenix.calcite.rel.PhoenixRel.ImplementorContext;
+import org.apache.phoenix.compile.ColumnProjector;
+import org.apache.phoenix.compile.ExpressionProjector;
+import org.apache.phoenix.compile.QueryPlan;
+import org.apache.phoenix.compile.RowProjector;
+import org.apache.phoenix.compile.SequenceManager;
+import org.apache.phoenix.compile.SequenceValueExpression;
+import org.apache.phoenix.compile.TupleProjectionCompiler;
+import org.apache.phoenix.coprocessor.MetaDataProtocol;
+import org.apache.phoenix.execute.RuntimeContext;
+import org.apache.phoenix.execute.TupleProjector;
+import org.apache.phoenix.expression.ColumnExpression;
+import org.apache.phoenix.expression.CorrelateVariableFieldAccessExpression;
+import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.parse.ParseNodeFactory;
+import org.apache.phoenix.parse.SequenceValueParseNode;
+import org.apache.phoenix.parse.TableName;
+import org.apache.phoenix.schema.ColumnRef;
+import org.apache.phoenix.schema.KeyValueSchema;
+import org.apache.phoenix.schema.PColumn;
+import org.apache.phoenix.schema.PColumnImpl;
+import org.apache.phoenix.schema.PName;
+import org.apache.phoenix.schema.PNameFactory;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableImpl;
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.schema.KeyValueSchema.KeyValueSchemaBuilder;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.util.SchemaUtil;
+
+import com.google.common.collect.Lists;
+
+public class PhoenixRelImplementorImpl implements PhoenixRel.Implementor {
+ private final RuntimeContext runtimeContext;
+ private TableRef tableRef;
+ private List<PColumn> mappedColumns;
+ private Stack<ImplementorContext> contextStack;
+ private SequenceManager sequenceManager;
+
+ public PhoenixRelImplementorImpl(RuntimeContext runtimeContext) {
+ this.runtimeContext = runtimeContext;
+ this.contextStack = new Stack<ImplementorContext>();
+ }
+
+ @Override
+ public QueryPlan visitInput(int i, PhoenixRel input) {
+ return input.implement(this);
+ }
+
+ @Override
+ public ColumnExpression newColumnExpression(int index) {
+ ColumnRef colRef = new ColumnRef(this.tableRef, this.mappedColumns.get(index).getPosition());
+ return colRef.newColumnExpression();
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public Expression newFieldAccessExpression(String variableId, int index, PDataType type) {
+ Expression fieldAccessExpr = runtimeContext.newCorrelateVariableReference(variableId, index);
+ return new CorrelateVariableFieldAccessExpression(runtimeContext, variableId, fieldAccessExpr);
+ }
+
+ @Override
+ public SequenceValueExpression newSequenceExpression(PhoenixSequence seq, SequenceValueParseNode.Op op) {
+ PName tenantName = seq.pc.getTenantId();
+ TableName tableName = TableName.create(seq.schemaName, seq.sequenceName);
+ try {
+ return sequenceManager.newSequenceReference(tenantName, tableName, null, op);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public RuntimeContext getRuntimeContext() {
+ return runtimeContext;
+ }
+
+ @Override
+ public void setTableRef(TableRef tableRef) {
+ this.tableRef = tableRef;
+ this.mappedColumns = PhoenixTable.getMappedColumns(tableRef.getTable());
+ }
+
+ @Override
+ public TableRef getTableRef() {
+ return this.tableRef;
+ }
+
+ @Override
+ public void setSequenceManager(SequenceManager sequenceManager) {
+ this.sequenceManager = sequenceManager;
+ }
+
+ @Override
+ public void pushContext(ImplementorContext context) {
+ this.contextStack.push(context);
+ }
+
+ @Override
+ public ImplementorContext popContext() {
+ return contextStack.pop();
+ }
+
+ @Override
+ public ImplementorContext getCurrentContext() {
+ return contextStack.peek();
+ }
+
+ @Override
+ public PTable createProjectedTable() {
+ List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList();
+ List<PColumn> columns = getCurrentContext().retainPKColumns ?
+ getTableRef().getTable().getColumns() : mappedColumns;
+ for (PColumn column : columns) {
+ sourceColumnRefs.add(new ColumnRef(getTableRef(), column.getPosition()));
+ }
+
+ try {
+ return TupleProjectionCompiler.createProjectedTable(getTableRef(), sourceColumnRefs, getCurrentContext().retainPKColumns);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public TupleProjector createTupleProjector() {
+ KeyValueSchemaBuilder builder = new KeyValueSchemaBuilder(0);
+ List<Expression> exprs = Lists.<Expression> newArrayList();
+ for (PColumn column : mappedColumns) {
+ if (!SchemaUtil.isPKColumn(column) || !getCurrentContext().retainPKColumns) {
+ Expression expr = new ColumnRef(tableRef, column.getPosition()).newColumnExpression();
+ exprs.add(expr);
+ builder.addField(expr);
+ }
+ }
+
+ return new TupleProjector(builder.build(), exprs.toArray(new Expression[exprs.size()]));
+ }
+
+ @Override
+ public RowProjector createRowProjector() {
+ List<ColumnProjector> columnProjectors = Lists.<ColumnProjector>newArrayList();
+ for (int i = 0; i < mappedColumns.size(); i++) {
+ PColumn column = mappedColumns.get(i);
+ Expression expr = newColumnExpression(i); // Do not use column.position() here.
+ columnProjectors.add(new ExpressionProjector(column.getName().getString(), getTableRef().getTable().getName().getString(), expr, false));
+ }
+ // TODO get estimate row size
+ return new RowProjector(columnProjectors, 0, false);
+ }
+
+ @Override
+ public TupleProjector project(List<Expression> exprs) {
+ KeyValueSchema.KeyValueSchemaBuilder builder = new KeyValueSchema.KeyValueSchemaBuilder(0);
+ List<PColumn> columns = Lists.<PColumn>newArrayList();
+ for (int i = 0; i < exprs.size(); i++) {
+ String name = ParseNodeFactory.createTempAlias();
+ Expression expr = exprs.get(i);
+ builder.addField(expr);
+ columns.add(new PColumnImpl(PNameFactory.newName(name), PNameFactory.newName(TupleProjector.VALUE_COLUMN_FAMILY),
+ expr.getDataType(), expr.getMaxLength(), expr.getScale(), expr.isNullable(),
- i, expr.getSortOrder(), null, null, false, name));
++ i, expr.getSortOrder(), null, null, false, name, false));
+ }
+ try {
+ PTable pTable = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME,
+ PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM,
+ null, null, columns, null, null, Collections.<PTable>emptyList(),
+ false, Collections.<PName>emptyList(), null, null, false, false, false, null,
- null, null, true);
++ null, null, true, false);
+ this.setTableRef(new TableRef(CalciteUtils.createTempAlias(), pTable, HConstants.LATEST_TIMESTAMP, false));
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+
+ return new TupleProjector(builder.build(), exprs.toArray(new Expression[exprs.size()]));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java
index 4003326,dac691f..62ccea1
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java
@@@ -231,8 -233,13 +233,18 @@@ public class ListJarsQueryPlan implemen
return false;
}
+ @Override
+ public QueryPlan limit(Integer limit) {
+ return this;
+ }
++
+ @Override
+ public Set<TableRef> getSourceRefs() {
+ return Collections.<TableRef>emptySet();
+ }
+
+ @Override
+ public Operation getOperation() {
+ return stmt.getUpdateOperation();
+ }
-}
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/compile/OrderByCompiler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
index fa95d88,4d343f3..2c5cd2f
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
@@@ -40,7 -46,7 +46,8 @@@ import org.apache.phoenix.util.ScanUtil
import org.apache.phoenix.util.ScanUtil.BytesComparator;
import org.apache.phoenix.util.SchemaUtil;
+import com.google.common.base.Objects;
+ import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
index ed0b335,551b05c..c1fc245
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
@@@ -178,8 -178,8 +178,8 @@@ public class TupleProjectionCompiler
null, table.getTimeStamp(), table.getSequenceNumber(), table.getPKName(),
retainPKColumns ? table.getBucketNum() : null, projectedColumns, null,
null, Collections.<PTable>emptyList(), table.isImmutableRows(), Collections.<PName>emptyList(), null, null,
- table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(),
- null, table.rowKeyOrderOptimizable(), table.isTransactional());
+ table.isWALDisabled(), retainPKColumns ? table.isMultiTenant() : false, table.getStoreNulls(), table.getViewType(),
- retainPKColumns ? table.getViewIndexId() : null, null, table.rowKeyOrderOptimizable());
++ retainPKColumns ? table.getViewIndexId() : null, null, table.rowKeyOrderOptimizable(), table.isTransactional());
}
// For extracting column references from single select statement
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java
index e6ec9aa,56e0ccd..1864251
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java
@@@ -112,4 -115,9 +119,9 @@@ public abstract class DelegateQueryPla
return delegate.useRoundRobinIterator();
}
+ @Override
+ public Operation getOperation() {
+ return delegate.getOperation();
+ }
+
-}
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
index bbac5a5,cf89380..30b3156
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
@@@ -117,15 -127,12 +126,20 @@@ public class HashJoinPlan extends Deleg
QueryServices.MAX_SERVER_CACHE_TIME_TO_LIVE_MS_ATTRIB, QueryServicesOptions.DEFAULT_MAX_SERVER_CACHE_TIME_TO_LIVE_MS);
}
+ public HashJoinInfo getJoinInfo() {
+ return this.joinInfo;
+ }
+
+ public SubPlan[] getSubPlans() {
+ return this.subPlans;
+ }
+
@Override
+ public Set<TableRef> getSourceRefs() {
+ return tableRefs;
+ }
+
+ @Override
public ResultIterator iterator() throws SQLException {
return iterator(DefaultParallelScanGrouper.getInstance());
}
@@@ -418,18 -431,13 +438,23 @@@
+ " IN (" + keyRangeRhsExpression.toString() + ")";
return Collections.<String> singletonList(step);
}
-
+
+
+ @Override
+ public QueryPlan getInnerPlan() {
+ return plan;
+ }
}
+
+ @Override
+ public QueryPlan limit(Integer limit) {
+ QueryPlan delegate = this.delegate.limit(limit);
+ if (delegate == this.delegate)
+ return this;
+
+ return new HashJoinPlan(this.statement, delegate, this.joinInfo,
+ this.subPlans, this.recompileWhereClause);
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
index c582cd2,41cae79..e353a5f
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
@@@ -646,13 -658,8 +658,16 @@@ public class SortMergeJoinPlan implemen
}
@Override
+ public QueryPlan limit(Integer limit) {
+ if (limit == null)
+ return this;
+
+ return new ClientScanPlan(this.getContext(), this.getStatement(), this.getTableRef(),
+ this.getProjector(), limit, null, OrderBy.EMPTY_ORDER_BY, this);
+ }
+
- }
+ public Set<TableRef> getSourceRefs() {
+ return tableRefs;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
index 134bafe,3208913..6bc7c18
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
@@@ -188,13 -202,18 +192,27 @@@ public class UnionPlan implements Query
return false;
}
+ @Override
+ public QueryPlan limit(Integer limit) {
+ if (limit == this.limit || (limit != null && limit.equals(this.limit)))
+ return this;
+
+ return new UnionPlan(this.parentContext, this.statement, this.tableRef, this.projector,
+ limit, this.orderBy, this.groupBy, this.plans, this.paramMetaData);
+ }
- }
+
+ @Override
+ public Operation getOperation() {
+ return statement.getOperation();
+ }
+
+ @Override
+ public Set<TableRef> getSourceRefs() {
+ // TODO is this correct?
+ Set<TableRef> sources = Sets.newHashSetWithExpectedSize(plans.size());
+ for (QueryPlan plan : plans) {
+ sources.addAll(plan.getSourceRefs());
+ }
+ return sources;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/test/java/org/apache/phoenix/calcite/ToExpressionTest.java
----------------------------------------------------------------------
diff --cc phoenix-core/src/test/java/org/apache/phoenix/calcite/ToExpressionTest.java
index 3734b4c,0000000..92a2479
mode 100644,000000..100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/calcite/ToExpressionTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/calcite/ToExpressionTest.java
@@@ -1,220 -1,0 +1,220 @@@
+package org.apache.phoenix.calcite;
+
+import static org.junit.Assert.assertEquals;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.schema.Function;
+import org.apache.calcite.schema.Schema;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.Table;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.tools.FrameworkConfig;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.Planner;
+import org.apache.phoenix.calcite.rel.PhoenixRelImplementorImpl;
+import org.apache.phoenix.calcite.rel.PhoenixRel.Implementor;
+import org.apache.phoenix.compile.ColumnResolver;
+import org.apache.phoenix.compile.ExpressionCompiler;
+import org.apache.phoenix.compile.FromCompiler;
+import org.apache.phoenix.compile.StatementContext;
+import org.apache.phoenix.execute.RuntimeContextImpl;
+import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.parse.ParseNode;
+import org.apache.phoenix.parse.SQLParser;
+import org.apache.phoenix.parse.SelectStatement;
+import org.apache.phoenix.query.BaseConnectionlessQueryTest;
+import org.apache.phoenix.schema.PName;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.TableRef;
+import org.junit.Test;
+
+
+public class ToExpressionTest extends BaseConnectionlessQueryTest {
+
+ @Test
+ public void toExpressionTest() throws Exception {
+ new ExpressionChecker(
+ "T",
+ "CREATE TABLE t(k1 VARCHAR PRIMARY KEY, k2 VARCHAR, v1 VARCHAR)",
+ "SELECT * FROM T WHERE K2 = 'foo'",
+ new WhereExpressionGetter())
+ .checkExpressionEquality()
+ .checkTypeEquality();
+ }
+
+ private static class ExpressionChecker {
+ private final PTable table;
+ private final Expression phoenixExpr;
+ private final RexNode calciteExpr;
+
+ public ExpressionChecker(String tableName, String ddl, String sql, ExpressionGetter getter) throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ PhoenixConnection pc = conn.unwrap(PhoenixConnection.class);
+ PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class);
+
+ conn.createStatement().execute(ddl);
- this.table = pc.getMetaDataCache().getTable(new PTableKey(null,tableName));
++ this.table = pc.getMetaDataCache().getTableRef(new PTableKey(null,tableName)).getTable();
+ SelectStatement select = new SQLParser(sql).parseQuery();
+ ColumnResolver resolver = FromCompiler.getResolverForQuery(select, pc);
+ this.phoenixExpr = getter.getParseNode(select).accept(new ExpressionCompiler(new StatementContext(stmt, resolver)));
+
+ Planner planner = getPlanner(pc, table);
+ SqlNode sqlNode = planner.parse(sql);
+ sqlNode = planner.validate(sqlNode);
+ RelNode rel = planner.convert(sqlNode);
+ this.calciteExpr = getter.getRexNode(rel);
+ }
+
+ public ExpressionChecker checkExpressionEquality() {
+ Implementor implementor = new PhoenixRelImplementorImpl(new RuntimeContextImpl());
+ implementor.setTableRef(new TableRef(table));
+ Expression e = CalciteUtils.toExpression(this.calciteExpr, implementor);
+ assertEquals(this.phoenixExpr,e);
+ return this;
+ }
+
+ public ExpressionChecker checkTypeEquality() {
+ // TODO
+ return this;
+ }
+ }
+
+ private static interface ExpressionGetter {
+ ParseNode getParseNode(SelectStatement select);
+ RexNode getRexNode(RelNode rel);
+ }
+
+ private static class WhereExpressionGetter implements ExpressionGetter {
+ @Override
+ public ParseNode getParseNode(SelectStatement select) {
+ return select.getWhere();
+ }
+ @Override
+ public RexNode getRexNode(RelNode rel) {
+ Filter filter = null;
+ while (filter == null && rel != null) {
+ if (rel instanceof Filter) {
+ filter = (Filter) rel;
+ } else if (!rel.getInputs().isEmpty()) {
+ rel = rel.getInput(0);
+ } else {
+ rel = null;
+ }
+ }
+ if (filter == null) {
+ throw new RuntimeException("Couldn't find Filter rel.");
+ }
+
+ return filter.getCondition();
+ }
+ }
+
+ private static Planner getPlanner(PhoenixConnection pc, PTable... tables) {
+ final SchemaPlus rootSchema = Frameworks.createRootSchema(true);
+ final FrameworkConfig config = Frameworks.newConfigBuilder()
+ .parserConfig(SqlParser.Config.DEFAULT)
+ .defaultSchema(rootSchema.add("phoenix",
+ TestSchema.create(pc, tables)))
+ .build();
+ return Frameworks.getPlanner(config);
+ }
+
+ private static class TestSchema implements Schema {
+ private final PhoenixConnection pc;
+ private final Map<String, Map<String, PTable>> tableMap;
+
+ public static TestSchema create(final PhoenixConnection pc, final PTable[] tables) {
+ Map<String, Map<String, PTable>> tableMap = new HashMap<String, Map<String, PTable>>();
+ for (PTable table : tables) {
+ PName schemaName = table.getSchemaName();
+ PName tableName = table.getTableName();
+ String schemaStr = schemaName == null ? "" : schemaName.getString();
+ Map<String, PTable> subSchemaMap = tableMap.get(schemaStr);
+ if (subSchemaMap == null) {
+ subSchemaMap = new HashMap<String, PTable>();
+ tableMap.put(schemaStr, subSchemaMap);
+ }
+ subSchemaMap.put(tableName.getString(), table);
+ }
+ return new TestSchema(pc, tableMap);
+ }
+
+ private TestSchema(final PhoenixConnection pc, final Map<String, Map<String, PTable>> tableMap) {
+ this.pc = pc;
+ this.tableMap = tableMap;
+ }
+
+ @Override
+ public Table getTable(String name) {
+ Map<String, PTable> rootTables = tableMap.get("");
+ if (rootTables == null)
+ return null;
+
+ PTable table = rootTables.get(name);
+ return new PhoenixTable(pc, table);
+ }
+
+ @Override
+ public Set<String> getTableNames() {
+ Map<String, PTable> rootTables = tableMap.get("");
+ return rootTables == null ? Collections.<String>emptySet() : rootTables.keySet();
+ }
+
+ @Override
+ public Collection<Function> getFunctions(String name) {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public Set<String> getFunctionNames() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public Schema getSubSchema(String name) {
+ Map<String, PTable> subMap = tableMap.get(name);
+ if (subMap == null)
+ return null;
+
+ Map<String, Map<String, PTable>> tableMap = new HashMap<String, Map<String, PTable>>();
+ tableMap.put("", subMap);
+ return new TestSchema(pc, tableMap);
+ }
+
+ @Override
+ public Set<String> getSubSchemaNames() {
+ return tableMap.keySet();
+ }
+
+ @Override
+ public org.apache.calcite.linq4j.tree.Expression getExpression(
+ SchemaPlus parentSchema, String name) {
+ return null;
+ }
+
+ @Override
+ public boolean isMutable() {
+ return false;
+ }
+
+ @Override
+ public boolean contentsHaveChangedSince(long lastCheck, long now) {
+ return false;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5a833c32/pom.xml
----------------------------------------------------------------------
diff --cc pom.xml
index 079c6c6,148bb51..c339cd8
--- a/pom.xml
+++ b/pom.xml
@@@ -109,7 -110,9 +110,9 @@@
<collections.version>3.2.1</collections.version>
<jodatime.version>2.7</jodatime.version>
<joni.version>2.1.2</joni.version>
- <calcite.version>1.5.0</calcite.version>
+ <calcite.version>1.6.0-SNAPSHOT</calcite.version>
+ <jettyVersion>8.1.7.v20120910</jettyVersion>
+ <tephra.version>0.6.3</tephra.version>
<!-- Test Dependencies -->
<mockito-all.version>1.8.5</mockito-all.version>
[40/52] [abbrv] phoenix git commit: PHOENIX-2477 ClassCastException
in IndexedWALEditCodec after HBASE-14501 (possible dataloss)
Posted by ma...@apache.org.
PHOENIX-2477 ClassCastException in IndexedWALEditCodec after HBASE-14501 (possible dataloss)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/9478d1f2
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/9478d1f2
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/9478d1f2
Branch: refs/heads/calcite
Commit: 9478d1f2b4faef9f698c0c7ffa6654f67b8253e5
Parents: 1c2b9b0
Author: Enis Soztutar <en...@apache.org>
Authored: Thu Dec 3 19:21:46 2015 -0800
Committer: Enis Soztutar <en...@apache.org>
Committed: Thu Dec 3 19:21:46 2015 -0800
----------------------------------------------------------------------
.../regionserver/wal/IndexedWALEditCodec.java | 56 ++++++++++++++++----
1 file changed, 46 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9478d1f2/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedWALEditCodec.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedWALEditCodec.java b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedWALEditCodec.java
index c29f77d..2534b34 100644
--- a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedWALEditCodec.java
+++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedWALEditCodec.java
@@ -19,7 +19,9 @@
package org.apache.hadoop.hbase.regionserver.wal;
import java.io.DataInput;
+import java.io.DataInputStream;
import java.io.DataOutput;
+import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
@@ -80,9 +82,35 @@ public class IndexedWALEditCodec extends WALCellCodec {
}
/**
+ * Returns a DataInput given an InputStream
+ */
+ private static DataInput getDataInput(InputStream is) {
+ return is instanceof DataInput
+ ? (DataInput) is
+ : new DataInputStream(is);
+ }
+
+ /**
+ * Returns a DataOutput given an OutputStream
+ */
+ private static DataOutput getDataOutput(OutputStream os) {
+ return os instanceof DataOutput
+ ? (DataOutput) os
+ : new DataOutputStream(os);
+ }
+
+ private static abstract class PhoenixBaseDecoder extends BaseDecoder {
+ protected DataInput dataInput;
+ public PhoenixBaseDecoder(InputStream in) {
+ super(in);
+ dataInput = getDataInput(this.in);
+ }
+ }
+
+ /**
* Custom Decoder that can handle a stream of regular and indexed {@link KeyValue}s.
*/
- public class IndexKeyValueDecoder extends BaseDecoder {
+ public static class IndexKeyValueDecoder extends PhoenixBaseDecoder {
/**
* Create a Decoder on the given input stream with the given Decoder to parse
@@ -95,11 +123,11 @@ public class IndexedWALEditCodec extends WALCellCodec {
@Override
protected KeyValue parseCell() throws IOException{
- return KeyValueCodec.readKeyValue((DataInput) this.in);
+ return KeyValueCodec.readKeyValue(this.dataInput);
}
}
- public class CompressedIndexKeyValueDecoder extends BaseDecoder {
+ public static class CompressedIndexKeyValueDecoder extends PhoenixBaseDecoder {
private Decoder decoder;
@@ -133,7 +161,15 @@ public class IndexedWALEditCodec extends WALCellCodec {
}
// its an indexedKeyValue, so parse it out specially
- return KeyValueCodec.readKeyValue((DataInput) this.in);
+ return KeyValueCodec.readKeyValue(this.dataInput);
+ }
+ }
+
+ private static abstract class PhoenixBaseEncoder extends BaseEncoder {
+ protected DataOutput dataOutput;
+ public PhoenixBaseEncoder(OutputStream out) {
+ super(out);
+ dataOutput = getDataOutput(this.out);
}
}
@@ -141,7 +177,7 @@ public class IndexedWALEditCodec extends WALCellCodec {
* Encode {@link IndexedKeyValue}s via the {@link KeyValueCodec}. Does <b>not</b> support
* compression.
*/
- private static class IndexKeyValueEncoder extends BaseEncoder {
+ private static class IndexKeyValueEncoder extends PhoenixBaseEncoder {
public IndexKeyValueEncoder(OutputStream os) {
super(os);
}
@@ -157,7 +193,7 @@ public class IndexedWALEditCodec extends WALCellCodec {
checkFlushed();
// use the standard encoding mechanism
- KeyValueCodec.write((DataOutput) this.out, KeyValueUtil.ensureKeyValue(cell));
+ KeyValueCodec.write(this.dataOutput, KeyValueUtil.ensureKeyValue(cell));
}
}
@@ -166,7 +202,7 @@ public class IndexedWALEditCodec extends WALCellCodec {
* <b>not</b> compatible with the {@link IndexKeyValueDecoder} - one cannot intermingle compressed
* and uncompressed WALs that contain index entries.
*/
- private static class CompressedIndexKeyValueEncoder extends BaseEncoder {
+ private static class CompressedIndexKeyValueEncoder extends PhoenixBaseEncoder {
private Encoder compressedKvEncoder;
public CompressedIndexKeyValueEncoder(OutputStream os, Encoder compressedKvEncoder) {
@@ -184,20 +220,20 @@ public class IndexedWALEditCodec extends WALCellCodec {
public void write(Cell cell) throws IOException {
//make sure we are open
checkFlushed();
-
+
//write the special marker so we can figure out which kind of kv is it
int marker = IndexedWALEditCodec.REGULAR_KEY_VALUE_MARKER;
if (cell instanceof IndexedKeyValue) {
marker = KeyValueCodec.INDEX_TYPE_LENGTH_MARKER;
}
out.write(marker);
-
+
//then serialize based on the marker
if (marker == IndexedWALEditCodec.REGULAR_KEY_VALUE_MARKER) {
this.compressedKvEncoder.write(cell);
}
else{
- KeyValueCodec.write((DataOutput) out, KeyValueUtil.ensureKeyValue(cell));
+ KeyValueCodec.write(this.dataOutput, KeyValueUtil.ensureKeyValue(cell));
}
}
}
[20/52] [abbrv] phoenix git commit: PHOENIX-2461 Tweak shutdown of
ConnectionQueryService during testing
Posted by ma...@apache.org.
PHOENIX-2461 Tweak shutdown of ConnectionQueryService during testing
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/33ed7d9f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/33ed7d9f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/33ed7d9f
Branch: refs/heads/calcite
Commit: 33ed7d9f0c59d53d7d8666642d18eb38b2d9f7b1
Parents: 07ede93
Author: James Taylor <jt...@salesforce.com>
Authored: Sun Nov 29 12:12:43 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Sun Nov 29 12:12:43 2015 -0800
----------------------------------------------------------------------
.../phoenix/end2end/ConnectionQueryServicesTestImpl.java | 10 ++++++++--
.../apache/phoenix/query/ConnectionQueryServicesImpl.java | 3 ++-
.../java/org/apache/phoenix/jdbc/PhoenixTestDriver.java | 2 +-
3 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/33ed7d9f/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
index d4f7c16..63dfaf9 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
@@ -61,10 +61,16 @@ public class ConnectionQueryServicesTestImpl extends ConnectionQueryServicesImpl
@Override
public void close() throws SQLException {
try {
- // Attempt to fix apparent memory leak...
clearCache();
} finally {
- super.close();
+ try {
+ // Since ConnectionQueryServicesTestImpl is a singleton
+ // during testing, attempt to stop the Zookeeper client
+ // service when the driver is closed.
+ if (zkClientService != null) zkClientService.stopAndWait();
+ } finally {
+ super.close();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/33ed7d9f/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 17df42d..7e1aee0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -220,6 +220,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private HConnection connection;
private TransactionServiceClient txServiceClient;
+ protected volatile ZKClientService zkClientService;
private volatile boolean initialized;
private volatile int nSequenceSaltBuckets;
@@ -305,7 +306,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private void initTxServiceClient() {
String zkQuorumServersString = connectionInfo.getZookeeperQuorum()+":"+connectionInfo.getPort();
- ZKClientService zkClientService = ZKClientServices.delegate(
+ zkClientService = ZKClientServices.delegate(
ZKClients.reWatchOnExpire(
ZKClients.retryOnFailure(
ZKClientService.Builder.of(zkQuorumServersString)
http://git-wip-us.apache.org/repos/asf/phoenix/blob/33ed7d9f/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
index f6df5c1..2591886 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
@@ -114,7 +114,7 @@ public class PhoenixTestDriver extends PhoenixEmbeddedDriver {
try {
queryServices.close();
} finally {
- queryServices.getExecutor().shutdown();
+ connectionQueryServices = null;
}
}
}
[02/52] [abbrv] phoenix git commit: PHOENIX-1674 Snapshot isolation
transaction support through Tephra (James Taylor, Thomas D'Silva)
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 21548b0..cb9b831 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -72,6 +72,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
@@ -169,6 +170,7 @@ import org.apache.phoenix.parse.SQLParser;
import org.apache.phoenix.parse.SelectStatement;
import org.apache.phoenix.parse.TableName;
import org.apache.phoenix.parse.UpdateStatisticsStatement;
+import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.ConnectionQueryServices.Feature;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
@@ -193,10 +195,13 @@ import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.StringUtil;
+import org.apache.phoenix.util.TransactionUtil;
import org.apache.phoenix.util.UpgradeUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import co.cask.tephra.TxConstants;
+
import com.google.common.base.Objects;
import com.google.common.collect.Iterators;
import com.google.common.collect.ListMultimap;
@@ -230,8 +235,9 @@ public class MetaDataClient {
VIEW_INDEX_ID + "," +
INDEX_TYPE + "," +
STORE_NULLS + "," +
- BASE_COLUMN_COUNT +
- ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+ BASE_COLUMN_COUNT + "," +
+ TRANSACTIONAL +
+ ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
private static final String CREATE_LINK =
"UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " +
TENANT_ID + "," +
@@ -372,7 +378,7 @@ public class MetaDataClient {
MetaDataMutationResult result = updateCache(schemaName, tableName, true);
return result.getMutationTime();
}
-
+
/**
* Update the cache with the latest as of the connection scn.
* @param schemaName
@@ -391,6 +397,10 @@ public class MetaDataClient {
public MetaDataMutationResult updateCache(PName tenantId, String schemaName, String tableName) throws SQLException {
return updateCache(tenantId, schemaName, tableName, false);
}
+
+ public MetaDataMutationResult updateCache(PName tenantId, String schemaName, String tableName, boolean alwaysHitServer) throws SQLException {
+ return updateCache(tenantId, schemaName, tableName, alwaysHitServer, null);
+ }
/**
* Update the cache with the latest as of the connection scn.
@@ -415,24 +425,47 @@ public class MetaDataClient {
long clientTimeStamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
return clientTimeStamp;
}
-
+
+ private long getCurrentScn() {
+ Long scn = connection.getSCN();
+ long currentScn = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
+ return currentScn;
+ }
+
private MetaDataMutationResult updateCache(PName tenantId, String schemaName, String tableName,
- boolean alwaysHitServer) throws SQLException { // TODO: pass byte[] herez
- long clientTimeStamp = getClientTimeStamp();
+ boolean alwaysHitServer, Long resolvedTimestamp) throws SQLException { // TODO: pass byte[] herez
boolean systemTable = SYSTEM_CATALOG_SCHEMA.equals(schemaName);
// System tables must always have a null tenantId
tenantId = systemTable ? null : tenantId;
PTable table = null;
+ PTableRef tableRef = null;
String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
long tableTimestamp = HConstants.LATEST_TIMESTAMP;
+ long tableResolvedTimestamp = HConstants.LATEST_TIMESTAMP;
try {
- table = connection.getTable(new PTableKey(tenantId, fullTableName));
+ tableRef = connection.getTableRef(new PTableKey(tenantId, fullTableName));
+ table = tableRef.getTable();
tableTimestamp = table.getTimeStamp();
+ tableResolvedTimestamp = tableRef.getResolvedTimeStamp();
} catch (TableNotFoundException e) {
}
- // Don't bother with server call: we can't possibly find a newer table
- if (table != null && !alwaysHitServer && (systemTable || tableTimestamp == clientTimeStamp - 1)) {
- return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS,QueryConstants.UNSET_TIMESTAMP,table);
+
+ boolean defaultTransactional = connection.getQueryServices().getProps().getBoolean(
+ QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB,
+ QueryServicesOptions.DEFAULT_TRANSACTIONAL);
+ // start a txn if all table are transactional by default or if we found the table in the cache and it is transactional
+ // TODO if system tables become transactional remove the check
+ boolean isTransactional = defaultTransactional || (table!=null && table.isTransactional());
+ if (!systemTable && isTransactional && !connection.getMutationState().isTransactionStarted()) {
+ connection.getMutationState().startTransaction();
+ }
+ resolvedTimestamp = resolvedTimestamp==null ? TransactionUtil.getResolvedTimestamp(connection, isTransactional, HConstants.LATEST_TIMESTAMP) : resolvedTimestamp;
+ // Do not make rpc to getTable if
+ // 1. table is a system table
+ // 2. table was already resolved as of that timestamp
+ if (table != null && !alwaysHitServer
+ && (systemTable || resolvedTimestamp == tableResolvedTimestamp)) {
+ return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, QueryConstants.UNSET_TIMESTAMP, table);
}
int maxTryCount = tenantId == null ? 1 : 2;
@@ -442,7 +475,12 @@ public class MetaDataClient {
do {
final byte[] schemaBytes = PVarchar.INSTANCE.toBytes(schemaName);
final byte[] tableBytes = PVarchar.INSTANCE.toBytes(tableName);
- result = connection.getQueryServices().getTable(tenantId, schemaBytes, tableBytes, tableTimestamp, clientTimeStamp);
+ ConnectionQueryServices queryServices = connection.getQueryServices();
+ result = queryServices.getTable(tenantId, schemaBytes, tableBytes, tableTimestamp, resolvedTimestamp);
+ // if the table was assumed to be transactional, but is actually not transactional then re-resolve as of the right timestamp (and vice versa)
+ if (table==null && result.getTable()!=null && result.getTable().isTransactional()!=isTransactional) {
+ result = queryServices.getTable(tenantId, schemaBytes, tableBytes, tableTimestamp, TransactionUtil.getResolvedTimestamp(connection, result.getTable().isTransactional(), HConstants.LATEST_TIMESTAMP));
+ }
if (SYSTEM_CATALOG_SCHEMA.equals(schemaName)) {
return result;
@@ -466,15 +504,20 @@ public class MetaDataClient {
// timestamp, we can handle this such that we don't ask the
// server again.
if (table != null) {
+ // Ensures that table in result is set to table found in our cache.
if (code == MutationCode.TABLE_ALREADY_EXISTS) {
- // Ensures that table in result is set to table found in our cache.
- result.setTable(table);
- // Although this table is up-to-date, the parent table may not be.
- // In this case, we update the parent table which may in turn pull
- // in indexes to add to this table.
- if (addIndexesFromPhysicalTable(result)) {
- connection.addTable(result.getTable());
- }
+ result.setTable(table);
+ // Although this table is up-to-date, the parent table may not be.
+ // In this case, we update the parent table which may in turn pull
+ // in indexes to add to this table.
+ long resolvedTime = TransactionUtil.getResolvedTime(connection, result);
+ if (addIndexesFromPhysicalTable(result, resolvedTimestamp)) {
+ connection.addTable(result.getTable(), resolvedTime);
+ }
+ else {
+ // if we aren't adding the table, we still need to update the resolved time of the table
+ connection.updateResolvedTimestamp(table, resolvedTime);
+ }
return result;
}
// If table was not found at the current time stamp and we have one cached, remove it.
@@ -566,10 +609,11 @@ public class MetaDataClient {
* of the table for which we just updated.
* TODO: combine this round trip with the one that updates the cache for the child table.
* @param result the result from updating the cache for the current table.
+ * @param resolvedTimestamp timestamp at which child table was resolved
* @return true if the PTable contained by result was modified and false otherwise
* @throws SQLException if the physical table cannot be found
*/
- private boolean addIndexesFromPhysicalTable(MetaDataMutationResult result) throws SQLException {
+ private boolean addIndexesFromPhysicalTable(MetaDataMutationResult result, Long resolvedTimestamp) throws SQLException {
PTable table = result.getTable();
// If not a view or if a view directly over an HBase table, there's nothing to do
if (table.getType() != PTableType.VIEW || table.getViewType() == ViewType.MAPPED) {
@@ -578,7 +622,7 @@ public class MetaDataClient {
String physicalName = table.getPhysicalName().getString();
String schemaName = SchemaUtil.getSchemaNameFromFullName(physicalName);
String tableName = SchemaUtil.getTableNameFromFullName(physicalName);
- MetaDataMutationResult parentResult = updateCache(null, schemaName, tableName, false);
+ MetaDataMutationResult parentResult = updateCache(null, schemaName, tableName, false, resolvedTimestamp);
PTable physicalTable = parentResult.getTable();
if (physicalTable == null) {
throw new TableNotFoundException(schemaName, tableName);
@@ -782,7 +826,7 @@ public class MetaDataClient {
public MutationState createTable(CreateTableStatement statement, byte[][] splits, PTable parent, String viewStatement, ViewType viewType, byte[][] viewColumnConstants, BitSet isViewColumnReferenced) throws SQLException {
PTable table = createTableInternal(statement, splits, parent, viewStatement, viewType, viewColumnConstants, isViewColumnReferenced, null, null);
- if (table == null || table.getType() == PTableType.VIEW) {
+ if (table == null || table.getType() == PTableType.VIEW || table.isTransactional()) {
return new MutationState(0,connection);
}
// Hack to get around the case when an SCN is specified on the connection.
@@ -1137,7 +1181,7 @@ public class MetaDataClient {
if (hbaseVersion < PhoenixDatabaseMetaData.MUTABLE_SI_VERSION_THRESHOLD) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_MUTABLE_INDEXES).setTableName(indexTableName.getTableName()).build().buildException();
}
- if (connection.getQueryServices().hasInvalidIndexConfiguration()) {
+ if (!connection.getQueryServices().isMutableIndexWALCodecInstalled() && !dataTable.isTransactional()) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_MUTABLE_INDEX_CONFIG).setTableName(indexTableName.getTableName()).build().buildException();
}
}
@@ -1268,19 +1312,19 @@ public class MetaDataClient {
// as there's no need to burn another sequence value.
if (allocateIndexId && indexId == null) {
Long scn = connection.getSCN();
- long timestamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
PName tenantId = connection.getTenantId();
String tenantIdStr = tenantId == null ? null : connection.getTenantId().getString();
PName physicalName = dataTable.getPhysicalName();
int nSequenceSaltBuckets = connection.getQueryServices().getSequenceSaltBuckets();
SequenceKey key = MetaDataUtil.getViewIndexSequenceKey(tenantIdStr, physicalName, nSequenceSaltBuckets);
- // Create at parent timestamp as we know that will be earlier than now
- // and earlier than any SCN if one is set.
+ // if scn is set create at scn-1, so we can see the sequence or else use latest timestamp (so that latest server time is used)
+ long sequenceTimestamp = scn!=null ? scn-1 : HConstants.LATEST_TIMESTAMP;
createSequence(key.getTenantId(), key.getSchemaName(), key.getSequenceName(),
true, Short.MIN_VALUE, 1, 1, false, Long.MIN_VALUE, Long.MAX_VALUE,
- dataTable.getTimeStamp());
+ sequenceTimestamp);
long[] seqValues = new long[1];
SQLException[] sqlExceptions = new SQLException[1];
+ long timestamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
connection.getQueryServices().incrementSequences(Collections.singletonList(new SequenceAllocation(key, 1)),
Math.max(timestamp, dataTable.getTimeStamp()), seqValues, sqlExceptions);
if (sqlExceptions[0] != null) {
@@ -1397,7 +1441,7 @@ public class MetaDataClient {
functionUpsert.setString(5, function.getJarPath());
functionUpsert.setString(6, function.getReturnType());
functionUpsert.execute();
- functionData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ functionData.addAll(connection.getMutationState().toMutations(null).next().getSecond());
connection.rollback();
MetaDataMutationResult result = connection.getQueryServices().createFunction(functionData, function, stmt.isTemporary());
MutationCode code = result.getMutationCode();
@@ -1498,29 +1542,36 @@ public class MetaDataClient {
long clientTimeStamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
boolean multiTenant = false;
boolean storeNulls = false;
+ boolean transactional = false;
Integer saltBucketNum = null;
String defaultFamilyName = null;
boolean isImmutableRows = false;
List<PName> physicalNames = Collections.emptyList();
boolean addSaltColumn = false;
boolean rowKeyOrderOptimizable = true;
+ Long timestamp = null;
if (parent != null && tableType == PTableType.INDEX) {
- // Index on view
- // TODO: Can we support a multi-tenant index directly on a multi-tenant
- // table instead of only a view? We don't have anywhere to put the link
- // from the table to the index, though.
- if (indexType == IndexType.LOCAL || (parent.getType() == PTableType.VIEW && parent.getViewType() != ViewType.MAPPED)) {
- PName physicalName = parent.getPhysicalName();
- saltBucketNum = parent.getBucketNum();
- addSaltColumn = (saltBucketNum != null && indexType != IndexType.LOCAL);
- defaultFamilyName = parent.getDefaultFamilyName() == null ? null : parent.getDefaultFamilyName().getString();
- if (indexType == IndexType.LOCAL) {
- saltBucketNum = null;
- // Set physical name of local index table
- physicalNames = Collections.singletonList(PNameFactory.newName(MetaDataUtil.getLocalIndexPhysicalName(physicalName.getBytes())));
- } else {
- // Set physical name of view index table
- physicalNames = Collections.singletonList(PNameFactory.newName(MetaDataUtil.getViewIndexPhysicalName(physicalName.getBytes())));
+ transactional = parent.isTransactional();
+ timestamp = TransactionUtil.getTableTimestamp(connection, transactional);
+ storeNulls = parent.getStoreNulls();
+ if (tableType == PTableType.INDEX) {
+ // Index on view
+ // TODO: Can we support a multi-tenant index directly on a multi-tenant
+ // table instead of only a view? We don't have anywhere to put the link
+ // from the table to the index, though.
+ if (indexType == IndexType.LOCAL || (parent.getType() == PTableType.VIEW && parent.getViewType() != ViewType.MAPPED)) {
+ PName physicalName = parent.getPhysicalName();
+ saltBucketNum = parent.getBucketNum();
+ addSaltColumn = (saltBucketNum != null && indexType != IndexType.LOCAL);
+ defaultFamilyName = parent.getDefaultFamilyName() == null ? null : parent.getDefaultFamilyName().getString();
+ if (indexType == IndexType.LOCAL) {
+ saltBucketNum = null;
+ // Set physical name of local index table
+ physicalNames = Collections.singletonList(PNameFactory.newName(MetaDataUtil.getLocalIndexPhysicalName(physicalName.getBytes())));
+ } else {
+ // Set physical name of view index table
+ physicalNames = Collections.singletonList(PNameFactory.newName(MetaDataUtil.getViewIndexPhysicalName(physicalName.getBytes())));
+ }
}
}
@@ -1536,7 +1587,7 @@ public class MetaDataClient {
incrementStatement.execute();
// Get list of mutations and add to table meta data that will be passed to server
// to guarantee order. This row will always end up last
- tableMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ tableMetaData.addAll(connection.getMutationState().toMutations(timestamp).next().getSecond());
connection.rollback();
// Add row linking from data table row to index table row
@@ -1561,12 +1612,10 @@ public class MetaDataClient {
}
Map<String,Object> tableProps = Maps.newHashMapWithExpectedSize(statement.getProps().size());
- Map<String,Object> commonFamilyProps = Collections.emptyMap();
+ Map<String,Object> commonFamilyProps = Maps.newHashMapWithExpectedSize(statement.getProps().size() + 1);
// Somewhat hacky way of determining if property is for HColumnDescriptor or HTableDescriptor
HColumnDescriptor defaultDescriptor = new HColumnDescriptor(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES);
if (!statement.getProps().isEmpty()) {
- commonFamilyProps = Maps.newHashMapWithExpectedSize(statement.getProps().size());
-
Collection<Pair<String,Object>> props = statement.getProps().get(QueryConstants.ALL_FAMILY_PROPERTIES_KEY);
for (Pair<String,Object> prop : props) {
if (defaultDescriptor.getValue(prop.getFirst()) == null) {
@@ -1580,7 +1629,7 @@ public class MetaDataClient {
// Although unusual, it's possible to set a mapped VIEW as having immutable rows.
// This tells Phoenix that you're managing the index maintenance yourself.
if (tableType != PTableType.INDEX && (tableType != PTableType.VIEW || viewType == ViewType.MAPPED)) {
- Boolean isImmutableRowsProp = (Boolean) tableProps.remove(PTable.IS_IMMUTABLE_ROWS_PROP_NAME);
+ Boolean isImmutableRowsProp = (Boolean) tableProps.get(PTable.IS_IMMUTABLE_ROWS_PROP_NAME);
if (isImmutableRowsProp == null) {
isImmutableRows = connection.getQueryServices().getProps().getBoolean(QueryServices.IMMUTABLE_ROWS_ATTRIB, QueryServicesOptions.DEFAULT_IMMUTABLE_ROWS);
} else {
@@ -1590,7 +1639,7 @@ public class MetaDataClient {
// Can't set any of these on views or shared indexes on views
if (tableType != PTableType.VIEW && indexId == null) {
- saltBucketNum = (Integer) tableProps.remove(PhoenixDatabaseMetaData.SALT_BUCKETS);
+ saltBucketNum = (Integer) tableProps.get(PhoenixDatabaseMetaData.SALT_BUCKETS);
if (saltBucketNum != null) {
if (saltBucketNum < 0 || saltBucketNum > SaltingUtil.MAX_BUCKET_NUM) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_BUCKET_NUM).build().buildException();
@@ -1607,42 +1656,110 @@ public class MetaDataClient {
addSaltColumn = (saltBucketNum != null);
}
- boolean removedProp = false;
// Can't set MULTI_TENANT or DEFAULT_COLUMN_FAMILY_NAME on an INDEX or a non mapped VIEW
if (tableType != PTableType.INDEX && (tableType != PTableType.VIEW || viewType == ViewType.MAPPED)) {
- Boolean multiTenantProp = (Boolean) tableProps.remove(PhoenixDatabaseMetaData.MULTI_TENANT);
+ Boolean multiTenantProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.MULTI_TENANT);
multiTenant = Boolean.TRUE.equals(multiTenantProp);
- defaultFamilyName = (String)tableProps.remove(PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME);
- removedProp = (defaultFamilyName != null);
+ defaultFamilyName = (String)tableProps.get(PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME);
}
boolean disableWAL = false;
- Boolean disableWALProp = (Boolean) tableProps.remove(PhoenixDatabaseMetaData.DISABLE_WAL);
+ Boolean disableWALProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.DISABLE_WAL);
if (disableWALProp != null) {
disableWAL = disableWALProp;
}
- Boolean storeNullsProp = (Boolean) tableProps.remove(PhoenixDatabaseMetaData.STORE_NULLS);
- storeNulls = storeNullsProp == null
- ? connection.getQueryServices().getProps().getBoolean(
- QueryServices.DEFAULT_STORE_NULLS_ATTRIB,
- QueryServicesOptions.DEFAULT_STORE_NULLS)
- : storeNullsProp;
+ Boolean storeNullsProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.STORE_NULLS);
+ if (storeNullsProp == null) {
+ if (parent == null) {
+ storeNulls = connection.getQueryServices().getProps().getBoolean(
+ QueryServices.DEFAULT_STORE_NULLS_ATTRIB,
+ QueryServicesOptions.DEFAULT_STORE_NULLS);
+ tableProps.put(PhoenixDatabaseMetaData.STORE_NULLS, Boolean.valueOf(storeNulls));
+ }
+ } else {
+ storeNulls = storeNullsProp;
+ }
+ Boolean transactionalProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.TRANSACTIONAL);
+ if (transactionalProp != null && parent != null) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.ONLY_TABLE_MAY_BE_DECLARED_TRANSACTIONAL)
+ .setSchemaName(schemaName).setTableName(tableName)
+ .build().buildException();
+ }
+ if (parent == null) {
+ if (transactionalProp == null) {
+ transactional = connection.getQueryServices().getProps().getBoolean(
+ QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB,
+ QueryServicesOptions.DEFAULT_TRANSACTIONAL);
+ } else {
+ transactional = transactionalProp;
+ }
+ }
+ tableProps.put(PhoenixDatabaseMetaData.TRANSACTIONAL, transactional);
+ if (transactional) {
+ // If TTL set, use Tephra TTL property name instead
+ Object ttl = commonFamilyProps.remove(HColumnDescriptor.TTL);
+ if (ttl != null) {
+ commonFamilyProps.put(TxConstants.PROPERTY_TTL, ttl);
+ }
+ }
+
+ boolean sharedTable = statement.getTableType() == PTableType.VIEW || indexId != null;
+ if (transactional) {
+ // Tephra uses an empty value cell as its delete marker, so we need to turn on
+ // storeNulls for transactional tables.
+ // If we use regular column delete markers (which is what non transactional tables
+ // use), then they get converted
+ // on the server, but this can mess up our secondary index code as the changes get
+ // committed prior to the
+ // maintenance code being able to see the prior state to update the rows correctly.
+ if (Boolean.FALSE.equals(storeNullsProp)) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.STORE_NULLS_MUST_BE_TRUE_FOR_TRANSACTIONAL)
+ .setSchemaName(schemaName).setTableName(tableName)
+ .build().buildException();
+ }
+ // Force STORE_NULLS to true when transactional as Tephra cannot deal with column deletes
+ storeNulls = true;
+ tableProps.put(PhoenixDatabaseMetaData.STORE_NULLS, Boolean.TRUE);
+
+ if (!sharedTable) {
+ Integer maxVersionsProp = (Integer) commonFamilyProps.get(HConstants.VERSIONS);
+ if (maxVersionsProp == null) {
+ if (parent != null) {
+ HTableDescriptor desc = connection.getQueryServices().getTableDescriptor(parent.getPhysicalName().getBytes());
+ if (desc != null) {
+ maxVersionsProp = desc.getFamily(SchemaUtil.getEmptyColumnFamily(parent)).getMaxVersions();
+ }
+ }
+ if (maxVersionsProp == null) {
+ maxVersionsProp = connection.getQueryServices().getProps().getInt(
+ QueryServices.MAX_VERSIONS_TRANSACTIONAL_ATTRIB,
+ QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL);
+ }
+ commonFamilyProps.put(HConstants.VERSIONS, maxVersionsProp);
+ }
+ }
+ }
+ timestamp = timestamp==null ? TransactionUtil.getTableTimestamp(connection, transactional) : timestamp;
// Delay this check as it is supported to have IMMUTABLE_ROWS and SALT_BUCKETS defined on views
- if ((statement.getTableType() == PTableType.VIEW || indexId != null) && !tableProps.isEmpty()) {
- throw new SQLExceptionInfo.Builder(SQLExceptionCode.VIEW_WITH_PROPERTIES).build()
- .buildException();
- }
- if (removedProp) {
- tableProps.put(PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME, defaultFamilyName);
+ if (sharedTable) {
+ if (tableProps.get(PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME) != null) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.DEFAULT_COLUMN_FAMILY_ON_SHARED_TABLE)
+ .setSchemaName(schemaName).setTableName(tableName)
+ .build().buildException();
+ }
+ if (SchemaUtil.hasHTableDescriptorProps(tableProps)) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.VIEW_WITH_PROPERTIES).build()
+ .buildException();
+ }
}
List<ColumnDef> colDefs = statement.getColumnDefs();
List<PColumn> columns;
LinkedHashSet<PColumn> pkColumns;
- if (tenantId != null && (tableType != PTableType.VIEW && indexId == null)) {
+ if (tenantId != null && !sharedTable) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_CREATE_TENANT_SPECIFIC_TABLE)
.setSchemaName(schemaName).setTableName(tableName).build().buildException();
}
@@ -1866,8 +1983,8 @@ public class MetaDataClient {
Collections.<PTable>emptyList(), isImmutableRows,
Collections.<PName>emptyList(), defaultFamilyName == null ? null :
PNameFactory.newName(defaultFamilyName), null,
- Boolean.TRUE.equals(disableWAL), false, false, null, indexId, indexType, true);
- connection.addTable(table);
+ Boolean.TRUE.equals(disableWAL), false, false, null, indexId, indexType, true, false);
+ connection.addTable(table, MetaDataProtocol.MIN_TABLE_TIMESTAMP);
} else if (tableType == PTableType.INDEX && indexId == null) {
if (tableProps.get(HTableDescriptor.MAX_FILESIZE) == null) {
int nIndexRowKeyColumns = isPK ? 1 : pkColumnsNames.size();
@@ -1922,7 +2039,7 @@ public class MetaDataClient {
addColumnMutation(schemaName, tableName, column, colUpsert, parentTableName, pkName, keySeq, saltBucketNum != null);
}
- tableMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ tableMetaData.addAll(connection.getMutationState().toMutations(timestamp).next().getSecond());
connection.rollback();
String dataTableName = parent == null || tableType == PTableType.VIEW ? null : parent.getTableName().getString();
@@ -1968,9 +2085,10 @@ public class MetaDataClient {
} else {
tableUpsert.setInt(20, BASE_TABLE_BASE_COLUMN_COUNT);
}
+ tableUpsert.setBoolean(21, transactional);
tableUpsert.execute();
- tableMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ tableMetaData.addAll(connection.getMutationState().toMutations(timestamp).next().getSecond());
connection.rollback();
/*
@@ -1996,7 +2114,9 @@ public class MetaDataClient {
MutationCode code = result.getMutationCode();
switch(code) {
case TABLE_ALREADY_EXISTS:
- addTableToCache(result);
+ if (result.getTable() != null) { // Can happen for transactional table that already exists as HBase table
+ addTableToCache(result);
+ }
if (!statement.ifNotExists()) {
throw new TableAlreadyExistsException(schemaName, tableName, result.getTable());
}
@@ -2016,11 +2136,11 @@ public class MetaDataClient {
default:
PName newSchemaName = PNameFactory.newName(schemaName);
PTable table = PTableImpl.makePTable(
- tenantId, newSchemaName, PNameFactory.newName(tableName), tableType, indexState, result.getMutationTime(),
+ tenantId, newSchemaName, PNameFactory.newName(tableName), tableType, indexState, timestamp!=null ? timestamp : result.getMutationTime(),
PTable.INITIAL_SEQ_NUM, pkName == null ? null : PNameFactory.newName(pkName), saltBucketNum, columns,
dataTableName == null ? null : newSchemaName, dataTableName == null ? null : PNameFactory.newName(dataTableName), Collections.<PTable>emptyList(), isImmutableRows,
physicalNames, defaultFamilyName == null ? null : PNameFactory.newName(defaultFamilyName), viewStatement, Boolean.TRUE.equals(disableWAL), multiTenant, storeNulls, viewType,
- indexId, indexType, rowKeyOrderOptimizable);
+ indexId, indexType, rowKeyOrderOptimizable, transactional);
result = new MetaDataMutationResult(code, result.getMutationTime(), table, true);
addTableToCache(result);
return table;
@@ -2175,6 +2295,8 @@ public class MetaDataClient {
MetaDataMutationResult result = connection.getQueryServices().dropTable(tableMetaData, tableType, cascade);
MutationCode code = result.getMutationCode();
+ PTable table = result.getTable();
+ boolean transactional = table!=null && table.isTransactional();
switch (code) {
case TABLE_NOT_FOUND:
if (!ifExists) { throw new TableNotFoundException(schemaName, tableName); }
@@ -2186,12 +2308,10 @@ public class MetaDataClient {
.setSchemaName(schemaName).setTableName(tableName).build().buildException();
default:
- connection.removeTable(tenantId, SchemaUtil.getTableName(schemaName, tableName), parentTableName,
- result.getMutationTime());
+ connection.removeTable(tenantId, SchemaUtil.getTableName(schemaName, tableName), parentTableName, result.getMutationTime());
if (result.getTable() != null && tableType != PTableType.VIEW) {
connection.setAutoCommit(true);
- PTable table = result.getTable();
boolean dropMetaData = result.getTable().getViewIndexId() == null &&
connection.getQueryServices().getProps().getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA);
long ts = (scn == null ? result.getMutationTime() : scn);
@@ -2334,12 +2454,12 @@ public class MetaDataClient {
return mutationCode;
}
- private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta) throws SQLException {
- return incrementTableSeqNum(table, expectedType, columnCountDelta, null, null, null, null);
+ private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta, Boolean isTransactional) throws SQLException {
+ return incrementTableSeqNum(table, expectedType, columnCountDelta, isTransactional, null, null, null, null);
}
private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta,
- Boolean isImmutableRows, Boolean disableWAL, Boolean isMultiTenant, Boolean storeNulls)
+ Boolean isTransactional, Boolean isImmutableRows, Boolean disableWAL, Boolean isMultiTenant, Boolean storeNulls)
throws SQLException {
String schemaName = table.getSchemaName().getString();
String tableName = table.getTableName().getString();
@@ -2371,6 +2491,9 @@ public class MetaDataClient {
if (storeNulls != null) {
mutateBooleanProperty(tenantId, schemaName, tableName, STORE_NULLS, storeNulls);
}
+ if (isTransactional != null) {
+ mutateBooleanProperty(tenantId, schemaName, tableName, TRANSACTIONAL, isTransactional);
+ }
return seqNum;
}
@@ -2404,10 +2527,13 @@ public class MetaDataClient {
Boolean multiTenantProp = null;
Boolean disableWALProp = null;
Boolean storeNullsProp = null;
+ Boolean isTransactionalProp = null;
ListMultimap<String,Pair<String,Object>> stmtProperties = statement.getProps();
Map<String, List<Pair<String, Object>>> properties = new HashMap<>(stmtProperties.size());
- PTable table = FromCompiler.getResolver(statement, connection).getTables().get(0).getTable();
+ TableRef tableRef = FromCompiler.getResolver(statement, connection).getTables().get(0);
+ PTable table = tableRef.getTable();
+ Long timeStamp = table.isTransactional() ? tableRef.getTimeStamp() : null;
List<ColumnDef> columnDefs = statement.getColumnDefs();
if (columnDefs == null) {
columnDefs = Collections.emptyList();
@@ -2426,6 +2552,8 @@ public class MetaDataClient {
disableWALProp = (Boolean)prop.getSecond();
} else if (propName.equals(STORE_NULLS)) {
storeNullsProp = (Boolean)prop.getSecond();
+ } else if (propName.equals(TRANSACTIONAL)) {
+ isTransactionalProp = (Boolean)prop.getSecond();
}
}
}
@@ -2433,6 +2561,7 @@ public class MetaDataClient {
}
boolean retried = false;
boolean changingPhoenixTableProperty = false;
+ boolean nonTxToTx = false;
while (true) {
ColumnResolver resolver = FromCompiler.getResolver(statement, connection);
table = resolver.getTables().get(0).getTable();
@@ -2487,6 +2616,23 @@ public class MetaDataClient {
changingPhoenixTableProperty = true;
}
}
+ Boolean isTransactional = null;
+ if (isTransactionalProp != null) {
+ if (isTransactionalProp.booleanValue() != table.isTransactional()) {
+ isTransactional = isTransactionalProp;
+ // We can only go one way: from non transactional to transactional
+ // Going the other way would require rewriting the cell timestamps
+ // and doing a major compaction to get rid of any Tephra specific
+ // delete markers.
+ if (!isTransactional) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.TX_MAY_NOT_SWITCH_TO_NON_TX)
+ .setSchemaName(schemaName).setTableName(tableName).build().buildException();
+ }
+ timeStamp = TransactionUtil.getTableTimestamp(connection, isTransactional);
+ changingPhoenixTableProperty = true;
+ nonTxToTx = true;
+ }
+ }
int numPkColumnsAdded = 0;
PreparedStatement colUpsert = connection.prepareStatement(INSERT_COLUMN_ALTER_TABLE);
@@ -2558,7 +2704,7 @@ public class MetaDataClient {
}
}
- columnMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ columnMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
connection.rollback();
} else {
// Check that HBase configured properly for mutable secondary indexing
@@ -2567,10 +2713,12 @@ public class MetaDataClient {
if (Boolean.FALSE.equals(isImmutableRows) && !table.getIndexes().isEmpty()) {
int hbaseVersion = connection.getQueryServices().getLowestClusterHBaseVersion();
if (hbaseVersion < PhoenixDatabaseMetaData.MUTABLE_SI_VERSION_THRESHOLD) {
- throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_MUTABLE_INDEXES).setSchemaName(schemaName).setTableName(tableName).build().buildException();
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_MUTABLE_INDEXES)
+ .setSchemaName(schemaName).setTableName(tableName).build().buildException();
}
- if (connection.getQueryServices().hasInvalidIndexConfiguration()) {
- throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_MUTABLE_INDEX_CONFIG).setSchemaName(schemaName).setTableName(tableName).build().buildException();
+ if (connection.getQueryServices().isMutableIndexWALCodecInstalled() && !table.isTransactional()) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_MUTABLE_INDEX_CONFIG)
+ .setSchemaName(schemaName).setTableName(tableName).build().buildException();
}
}
if (Boolean.TRUE.equals(multiTenant)) {
@@ -2578,17 +2726,17 @@ public class MetaDataClient {
}
}
- if (numPkColumnsAdded>0 && !table.getIndexes().isEmpty()) {
+ if (!table.getIndexes().isEmpty() && (numPkColumnsAdded>0 || nonTxToTx)) {
for (PTable index : table.getIndexes()) {
- incrementTableSeqNum(index, index.getType(), numPkColumnsAdded);
+ incrementTableSeqNum(index, index.getType(), numPkColumnsAdded, nonTxToTx ? Boolean.TRUE : null);
}
- tableMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
connection.rollback();
}
long seqNum = table.getSequenceNumber();
if (changingPhoenixTableProperty || columnDefs.size() > 0) {
- seqNum = incrementTableSeqNum(table, statement.getTableType(), columnDefs.size(), isImmutableRows, disableWAL, multiTenant, storeNulls);
- tableMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ seqNum = incrementTableSeqNum(table, statement.getTableType(), columnDefs.size(), isTransactional, isImmutableRows, disableWAL, multiTenant, storeNulls);
+ tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
connection.rollback();
}
@@ -2626,10 +2774,22 @@ public class MetaDataClient {
return new MutationState(0,connection);
}
- // Only update client side cache if we aren't adding a PK column to a table with indexes.
+ // Only update client side cache if we aren't adding a PK column to a table with indexes or
+ // transitioning a table from non transactional to transactional.
// We could update the cache manually then too, it'd just be a pain.
- if (numPkColumnsAdded==0 || table.getIndexes().isEmpty()) {
- connection.addColumn(tenantId, SchemaUtil.getTableName(schemaName, tableName), columns, result.getMutationTime(), seqNum, isImmutableRows == null ? table.isImmutableRows() : isImmutableRows, disableWAL == null ? table.isWALDisabled() : disableWAL, multiTenant == null ? table.isMultiTenant() : multiTenant, storeNulls == null ? table.getStoreNulls() : storeNulls);
+ if (table.getIndexes().isEmpty() || (numPkColumnsAdded==0 && !nonTxToTx)) {
+ connection.addColumn(
+ tenantId,
+ SchemaUtil.getTableName(schemaName, tableName),
+ columns,
+ result.getMutationTime(),
+ seqNum,
+ isImmutableRows == null ? table.isImmutableRows() : isImmutableRows,
+ disableWAL == null ? table.isWALDisabled() : disableWAL,
+ multiTenant == null ? table.isMultiTenant() : multiTenant,
+ storeNulls == null ? table.getStoreNulls() : storeNulls,
+ isTransactional == null ? table.isTransactional() : isTransactional,
+ TransactionUtil.getResolvedTime(connection, result));
}
// Delete rows in view index if we haven't dropped it already
// We only need to do this if the multiTenant transitioned to false
@@ -2760,12 +2920,12 @@ public class MetaDataClient {
boolean retried = false;
while (true) {
final ColumnResolver resolver = FromCompiler.getResolver(statement, connection);
- PTable table = resolver.getTables().get(0).getTable();
+ TableRef tableRef = resolver.getTables().get(0);
+ PTable table = tableRef.getTable();
List<ColumnName> columnRefs = statement.getColumnRefs();
if(columnRefs == null) {
columnRefs = Lists.newArrayListWithCapacity(0);
}
- TableRef tableRef = null;
List<ColumnRef> columnsToDrop = Lists.newArrayListWithExpectedSize(columnRefs.size() + table.getIndexes().size());
List<TableRef> indexesToDrop = Lists.newArrayListWithExpectedSize(table.getIndexes().size());
List<Mutation> tableMetaData = Lists.newArrayListWithExpectedSize((table.getIndexes().size() + 1) * (1 + table.getColumns().size() - columnRefs.size()));
@@ -2781,14 +2941,13 @@ public class MetaDataClient {
}
throw e;
}
- tableRef = columnRef.getTableRef();
PColumn columnToDrop = columnRef.getColumn();
tableColumnsToDrop.add(columnToDrop);
if (SchemaUtil.isPKColumn(columnToDrop)) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_DROP_PK)
.setColumnName(columnToDrop.getName().getString()).build().buildException();
}
- columnsToDrop.add(new ColumnRef(tableRef, columnToDrop.getPosition()));
+ columnsToDrop.add(new ColumnRef(columnRef.getTableRef(), columnToDrop.getPosition()));
}
dropColumnMutations(table, tableColumnsToDrop, tableMetaData);
@@ -2810,16 +2969,17 @@ public class MetaDataClient {
}
}
if(!indexColumnsToDrop.isEmpty()) {
- incrementTableSeqNum(index, index.getType(), -indexColumnsToDrop.size());
+ incrementTableSeqNum(index, index.getType(), -indexColumnsToDrop.size(), null);
dropColumnMutations(index, indexColumnsToDrop, tableMetaData);
}
}
- tableMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ Long timeStamp = table.isTransactional() ? tableRef.getTimeStamp() : null;
+ tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
connection.rollback();
- long seqNum = incrementTableSeqNum(table, statement.getTableType(), -tableColumnsToDrop.size());
- tableMetaData.addAll(connection.getMutationState().toMutations().next().getSecond());
+ long seqNum = incrementTableSeqNum(table, statement.getTableType(), -tableColumnsToDrop.size(), null);
+ tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
connection.rollback();
// Force table header to be first in list
Collections.reverse(tableMetaData);
@@ -2870,7 +3030,7 @@ public class MetaDataClient {
// client-side cache as it would be too painful. Just let it pull it over from
// the server when needed.
if (tableColumnsToDrop.size() > 0 && indexesToDrop.isEmpty()) {
- connection.removeColumn(tenantId, SchemaUtil.getTableName(schemaName, tableName) , tableColumnsToDrop, result.getMutationTime(), seqNum);
+ connection.removeColumn(tenantId, SchemaUtil.getTableName(schemaName, tableName) , tableColumnsToDrop, result.getMutationTime(), seqNum, TransactionUtil.getResolvedTime(connection, result));
}
// If we have a VIEW, then only delete the metadata, and leave the table data alone
if (table.getType() != PTableType.VIEW) {
@@ -2955,7 +3115,8 @@ public class MetaDataClient {
tableUpsert.close();
}
}
- List<Mutation> tableMetadata = connection.getMutationState().toMutations().next().getSecond();
+ Long timeStamp = indexRef.getTable().isTransactional() ? indexRef.getTimeStamp() : null;
+ List<Mutation> tableMetadata = connection.getMutationState().toMutations(timeStamp).next().getSecond();
connection.rollback();
MetaDataMutationResult result = connection.getQueryServices().updateIndexState(tableMetadata, dataTableName);
@@ -3003,9 +3164,9 @@ public class MetaDataClient {
}
private PTable addTableToCache(MetaDataMutationResult result) throws SQLException {
- addIndexesFromPhysicalTable(result);
+ addIndexesFromPhysicalTable(result, null);
PTable table = result.getTable();
- connection.addTable(table);
+ connection.addTable(table, TransactionUtil.getResolvedTime(connection, result));
return table;
}
@@ -3025,7 +3186,8 @@ public class MetaDataClient {
*/
boolean isSharedIndex = table.getViewIndexId() != null;
if (isSharedIndex) {
- return connection.getQueryServices().getTableStats(table.getPhysicalName().getBytes(), getClientTimeStamp());
+ // we are assuming the stats table is not transactional
+ return connection.getQueryServices().getTableStats(table.getPhysicalName().getBytes(), getCurrentScn());
}
boolean isView = table.getType() == PTableType.VIEW;
String physicalName = table.getPhysicalName().getString();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java
index f015177..a3103bf 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java
@@ -17,6 +17,8 @@
*/
package org.apache.phoenix.schema;
+import java.sql.SQLException;
+
import org.apache.phoenix.parse.PFunction;
import org.apache.phoenix.query.MetaDataMutated;
@@ -28,7 +30,7 @@ public interface PMetaData extends MetaDataMutated, Iterable<PTable>, Cloneable
}
public int size();
public PMetaData clone();
- public PTable getTable(PTableKey key) throws TableNotFoundException;
+ public PTableRef getTableRef(PTableKey key) throws TableNotFoundException;
public PMetaData pruneTables(Pruner pruner);
public PFunction getFunction(PTableKey key) throws FunctionNotFoundException;
public PMetaData pruneFunctions(Pruner pruner);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
index a15de13..9e4460d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
@@ -41,32 +41,12 @@ import com.google.common.primitives.Longs;
*
*/
public class PMetaDataImpl implements PMetaData {
- private static final class PTableRef {
- public final PTable table;
- public final int estSize;
- public volatile long lastAccessTime;
-
- public PTableRef(PTable table, long lastAccessTime, int estSize) {
- this.table = table;
- this.lastAccessTime = lastAccessTime;
- this.estSize = estSize;
- }
-
- public PTableRef(PTable table, long lastAccessTime) {
- this (table, lastAccessTime, table.getEstimatedSize());
- }
-
- public PTableRef(PTableRef tableRef) {
- this (tableRef.table, tableRef.lastAccessTime, tableRef.estSize);
- }
- }
-
private static class PMetaDataCache implements Cloneable {
private static final int MIN_REMOVAL_SIZE = 3;
private static final Comparator<PTableRef> COMPARATOR = new Comparator<PTableRef>() {
@Override
public int compare(PTableRef tableRef1, PTableRef tableRef2) {
- return Longs.compare(tableRef1.lastAccessTime, tableRef2.lastAccessTime);
+ return Longs.compare(tableRef1.getLastAccessTime(), tableRef2.getLastAccessTime());
}
};
private static final MinMaxPriorityQueue.Builder<PTableRef> BUILDER = MinMaxPriorityQueue.orderedBy(COMPARATOR);
@@ -97,7 +77,7 @@ public class PMetaDataImpl implements PMetaData {
Map<PTableKey,PTableRef> newTables = newMap(Math.max(tables.size(),expectedCapacity));
// Copy value so that access time isn't changing anymore
for (PTableRef tableAccess : tables.values()) {
- newTables.put(tableAccess.table.getKey(), new PTableRef(tableAccess));
+ newTables.put(tableAccess.getTable().getKey(), new PTableRef(tableAccess));
}
return newTables;
}
@@ -133,7 +113,7 @@ public class PMetaDataImpl implements PMetaData {
if (tableAccess == null) {
return null;
}
- tableAccess.lastAccessTime = timeKeeper.getCurrentTime();
+ tableAccess.setLastAccessTime(timeKeeper.getCurrentTime());
return tableAccess;
}
@@ -157,37 +137,37 @@ public class PMetaDataImpl implements PMetaData {
// Add to new cache, but track references to remove when done
// to bring cache at least overage amount below it's max size.
for (PTableRef tableRef : this.tables.values()) {
- newCache.put(tableRef.table.getKey(), new PTableRef(tableRef));
+ newCache.put(tableRef.getTable().getKey(), new PTableRef(tableRef));
toRemove.add(tableRef);
- toRemoveBytes += tableRef.estSize;
- while (toRemoveBytes - toRemove.peekLast().estSize >= overage) {
+ toRemoveBytes += tableRef.getEstSize();
+ while (toRemoveBytes - toRemove.peekLast().getEstSize() >= overage) {
PTableRef removedRef = toRemove.removeLast();
- toRemoveBytes -= removedRef.estSize;
+ toRemoveBytes -= removedRef.getEstSize();
}
}
for (PTableRef toRemoveRef : toRemove) {
- newCache.remove(toRemoveRef.table.getKey());
+ newCache.remove(toRemoveRef.getTable().getKey());
}
return newCache;
}
private PTable put(PTableKey key, PTableRef ref) {
- currentByteSize += ref.estSize;
+ currentByteSize += ref.getEstSize();
PTableRef oldTableAccess = this.tables.put(key, ref);
PTable oldTable = null;
if (oldTableAccess != null) {
- currentByteSize -= oldTableAccess.estSize;
- oldTable = oldTableAccess.table;
+ currentByteSize -= oldTableAccess.getEstSize();
+ oldTable = oldTableAccess.getTable();
}
return oldTable;
}
- public PTable put(PTableKey key, PTable value) {
- return put(key, new PTableRef(value, timeKeeper.getCurrentTime()));
+ public PTable put(PTableKey key, PTable value, long resolvedTime) {
+ return put(key, new PTableRef(value, timeKeeper.getCurrentTime(), resolvedTime));
}
- public PTable putDuplicate(PTableKey key, PTable value) {
- return put(key, new PTableRef(value, timeKeeper.getCurrentTime(), 0));
+ public PTable putDuplicate(PTableKey key, PTable value, long resolvedTime) {
+ return put(key, new PTableRef(value, timeKeeper.getCurrentTime(), 0, resolvedTime));
}
public PTable remove(PTableKey key) {
@@ -195,8 +175,8 @@ public class PMetaDataImpl implements PMetaData {
if (value == null) {
return null;
}
- currentByteSize -= value.estSize;
- return value.table;
+ currentByteSize -= value.getEstSize();
+ return value.getTable();
}
public Iterator<PTable> iterator() {
@@ -210,7 +190,7 @@ public class PMetaDataImpl implements PMetaData {
@Override
public PTable next() {
- return iterator.next().table;
+ return iterator.next().getTable();
}
@Override
@@ -254,12 +234,12 @@ public class PMetaDataImpl implements PMetaData {
}
@Override
- public PTable getTable(PTableKey key) throws TableNotFoundException {
+ public PTableRef getTableRef(PTableKey key) throws TableNotFoundException {
PTableRef ref = metaData.get(key);
if (ref == null) {
throw new TableNotFoundException(key.getName());
}
- return ref.table;
+ return ref;
}
@Override
@@ -276,22 +256,29 @@ public class PMetaDataImpl implements PMetaData {
return metaData.size();
}
+ @Override
+ public PMetaData updateResolvedTimestamp(PTable table, long resolvedTimestamp) throws SQLException {
+ PMetaDataCache clone = metaData.clone();
+ clone.putDuplicate(table.getKey(), table, resolvedTimestamp);
+ return new PMetaDataImpl(clone);
+ }
@Override
- public PMetaData addTable(PTable table) throws SQLException {
+ public PMetaData addTable(PTable table, long resolvedTime) throws SQLException {
int netGain = 0;
PTableKey key = table.getKey();
PTableRef oldTableRef = metaData.get(key);
if (oldTableRef != null) {
- netGain -= oldTableRef.estSize;
+ netGain -= oldTableRef.getEstSize();
}
PTable newParentTable = null;
+ long parentResolvedTimestamp = resolvedTime;
if (table.getParentName() != null) { // Upsert new index table into parent data table list
String parentName = table.getParentName().getString();
PTableRef oldParentRef = metaData.get(new PTableKey(table.getTenantId(), parentName));
// If parentTable isn't cached, that's ok we can skip this
if (oldParentRef != null) {
- List<PTable> oldIndexes = oldParentRef.table.getIndexes();
+ List<PTable> oldIndexes = oldParentRef.getTable().getIndexes();
List<PTable> newIndexes = Lists.newArrayListWithExpectedSize(oldIndexes.size() + 1);
newIndexes.addAll(oldIndexes);
for (int i = 0; i < newIndexes.size(); i++) {
@@ -302,8 +289,8 @@ public class PMetaDataImpl implements PMetaData {
}
}
newIndexes.add(table);
- netGain -= oldParentRef.estSize;
- newParentTable = PTableImpl.makePTable(oldParentRef.table, table.getTimeStamp(), newIndexes);
+ netGain -= oldParentRef.getEstSize();
+ newParentTable = PTableImpl.makePTable(oldParentRef.getTable(), table.getTimeStamp(), newIndexes);
netGain += newParentTable.getEstimatedSize();
}
}
@@ -314,24 +301,24 @@ public class PMetaDataImpl implements PMetaData {
PMetaDataCache newMetaData = overage <= 0 ? metaData.clone() : metaData.cloneMinusOverage(overage);
if (newParentTable != null) { // Upsert new index table into parent data table list
- newMetaData.put(newParentTable.getKey(), newParentTable);
- newMetaData.putDuplicate(table.getKey(), table);
+ newMetaData.put(newParentTable.getKey(), newParentTable, parentResolvedTimestamp);
+ newMetaData.putDuplicate(table.getKey(), table, resolvedTime);
} else {
- newMetaData.put(table.getKey(), table);
+ newMetaData.put(table.getKey(), table, resolvedTime);
}
for (PTable index : table.getIndexes()) {
- newMetaData.putDuplicate(index.getKey(), index);
+ newMetaData.putDuplicate(index.getKey(), index, resolvedTime);
}
return new PMetaDataImpl(newMetaData);
}
@Override
- public PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columnsToAdd, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls) throws SQLException {
+ public PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columnsToAdd, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException {
PTableRef oldTableRef = metaData.get(new PTableKey(tenantId, tableName));
if (oldTableRef == null) {
return this;
}
- List<PColumn> oldColumns = PTableImpl.getColumnsToClone(oldTableRef.table);
+ List<PColumn> oldColumns = PTableImpl.getColumnsToClone(oldTableRef.getTable());
List<PColumn> newColumns;
if (columnsToAdd.isEmpty()) {
newColumns = oldColumns;
@@ -340,8 +327,8 @@ public class PMetaDataImpl implements PMetaData {
newColumns.addAll(oldColumns);
newColumns.addAll(columnsToAdd);
}
- PTable newTable = PTableImpl.makePTable(oldTableRef.table, tableTimeStamp, tableSeqNum, newColumns, isImmutableRows, isWalDisabled, isMultitenant, storeNulls);
- return addTable(newTable);
+ PTable newTable = PTableImpl.makePTable(oldTableRef.getTable(), tableTimeStamp, tableSeqNum, newColumns, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional);
+ return addTable(newTable, resolvedTime);
}
@Override
@@ -368,7 +355,7 @@ public class PMetaDataImpl implements PMetaData {
}
// also remove its reference from parent table
if (parentTableRef != null) {
- List<PTable> oldIndexes = parentTableRef.table.getIndexes();
+ List<PTable> oldIndexes = parentTableRef.getTable().getIndexes();
if(oldIndexes != null && !oldIndexes.isEmpty()) {
List<PTable> newIndexes = Lists.newArrayListWithExpectedSize(oldIndexes.size());
newIndexes.addAll(oldIndexes);
@@ -377,13 +364,13 @@ public class PMetaDataImpl implements PMetaData {
if (index.getName().getString().equals(tableName)) {
newIndexes.remove(i);
PTable parentTable = PTableImpl.makePTable(
- parentTableRef.table,
- tableTimeStamp == HConstants.LATEST_TIMESTAMP ? parentTableRef.table.getTimeStamp() : tableTimeStamp,
+ parentTableRef.getTable(),
+ tableTimeStamp == HConstants.LATEST_TIMESTAMP ? parentTableRef.getTable().getTimeStamp() : tableTimeStamp,
newIndexes);
if (tables == null) {
tables = metaData.clone();
}
- tables.put(parentTable.getKey(), parentTable);
+ tables.put(parentTable.getKey(), parentTable, parentTableRef.getResolvedTimeStamp());
break;
}
}
@@ -393,12 +380,12 @@ public class PMetaDataImpl implements PMetaData {
}
@Override
- public PMetaData removeColumn(PName tenantId, String tableName, List<PColumn> columnsToRemove, long tableTimeStamp, long tableSeqNum) throws SQLException {
+ public PMetaData removeColumn(PName tenantId, String tableName, List<PColumn> columnsToRemove, long tableTimeStamp, long tableSeqNum, long resolvedTime) throws SQLException {
PTableRef tableRef = metaData.get(new PTableKey(tenantId, tableName));
if (tableRef == null) {
return this;
}
- PTable table = tableRef.table;
+ PTable table = tableRef.getTable();
PMetaDataCache tables = metaData.clone();
for (PColumn columnToRemove : columnsToRemove) {
PColumn column;
@@ -427,7 +414,7 @@ public class PMetaDataImpl implements PMetaData {
table = PTableImpl.makePTable(table, tableTimeStamp, tableSeqNum, columns);
}
- tables.put(table.getKey(), table);
+ tables.put(table.getKey(), table, resolvedTime);
return new PMetaDataImpl(tables);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
index a2979d4..ec97394 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
@@ -307,13 +307,14 @@ public interface PTable extends PMetaDataEntity {
PName getPhysicalName();
boolean isImmutableRows();
- void getIndexMaintainers(ImmutableBytesWritable ptr, PhoenixConnection connection);
+ boolean getIndexMaintainers(ImmutableBytesWritable ptr, PhoenixConnection connection);
IndexMaintainer getIndexMaintainer(PTable dataTable, PhoenixConnection connection);
PName getDefaultFamilyName();
boolean isWALDisabled();
boolean isMultiTenant();
boolean getStoreNulls();
+ boolean isTransactional();
ViewType getViewType();
String getViewStatement();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index ff4b512..0827ea7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -77,6 +77,8 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.sun.istack.NotNull;
+import co.cask.tephra.TxConstants;
+
/**
*
* Base class for PTable implementors. Provides abstraction for
@@ -123,6 +125,7 @@ public class PTableImpl implements PTable {
private boolean disableWAL;
private boolean multiTenant;
private boolean storeNulls;
+ private boolean isTransactional;
private ViewType viewType;
private Short viewIndexId;
private int estimatedSize;
@@ -202,7 +205,7 @@ public class PTableImpl implements PTable {
table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), getColumnsToClone(table), parentSchemaName, table.getParentTableName(),
indexes, table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), viewStatement,
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
- table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable());
+ table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional());
}
public static PTableImpl makePTable(PTable table, List<PColumn> columns) throws SQLException {
@@ -211,7 +214,7 @@ public class PTableImpl implements PTable {
table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(),
table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
- table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable());
+ table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional());
}
public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List<PColumn> columns) throws SQLException {
@@ -220,7 +223,7 @@ public class PTableImpl implements PTable {
sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(),
table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(),
table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(),
- table.getBaseColumnCount(), table.rowKeyOrderOptimizable());
+ table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional());
}
public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List<PColumn> columns, boolean isImmutableRows) throws SQLException {
@@ -229,16 +232,16 @@ public class PTableImpl implements PTable {
sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(),
table.getIndexes(), isImmutableRows, table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(),
- table.getIndexType(), table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable());
+ table.getIndexType(), table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional());
}
- public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List<PColumn> columns, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls) throws SQLException {
+ public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List<PColumn> columns, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional) throws SQLException {
return new PTableImpl(
table.getTenantId(), table.getSchemaName(), table.getTableName(), table.getType(), table.getIndexState(), timeStamp,
sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(),
table.getIndexes(), isImmutableRows, table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
isWalDisabled, isMultitenant, storeNulls, table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(),
- table.getBaseColumnCount(), table.rowKeyOrderOptimizable());
+ table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), isTransactional);
}
public static PTableImpl makePTable(PTable table, PIndexState state) throws SQLException {
@@ -248,7 +251,7 @@ public class PTableImpl implements PTable {
table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(),
table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
- table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable());
+ table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional());
}
public static PTableImpl makePTable(PTable table, boolean rowKeyOrderOptimizable) throws SQLException {
@@ -258,7 +261,7 @@ public class PTableImpl implements PTable {
table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(),
table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(),
- table.getBaseColumnCount(), rowKeyOrderOptimizable);
+ table.getBaseColumnCount(), rowKeyOrderOptimizable, table.isTransactional());
}
public static PTableImpl makePTable(PTable table, PTableStats stats) throws SQLException {
@@ -268,28 +271,28 @@ public class PTableImpl implements PTable {
table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(),
table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), stats,
- table.getBaseColumnCount(), table.rowKeyOrderOptimizable());
+ table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional());
}
public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, long timeStamp, long sequenceNumber,
PName pkName, Integer bucketNum, List<PColumn> columns, PName dataSchemaName, PName dataTableName, List<PTable> indexes,
boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant,
- boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, boolean rowKeyOrderOptimizable) throws SQLException {
+ boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, boolean rowKeyOrderOptimizable, boolean isTransactional) throws SQLException {
return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, dataSchemaName,
dataTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName,
viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId,
- indexType, PTableStats.EMPTY_STATS, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, rowKeyOrderOptimizable);
+ indexType, PTableStats.EMPTY_STATS, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, rowKeyOrderOptimizable, isTransactional);
}
public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type,
PIndexState state, long timeStamp, long sequenceNumber, PName pkName, Integer bucketNum,
List<PColumn> columns, PName dataSchemaName, PName dataTableName, List<PTable> indexes,
boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression,
- boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, @NotNull PTableStats stats, int baseColumnCount, boolean rowKeyOrderOptimizable)
+ boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, @NotNull PTableStats stats, int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional)
throws SQLException {
return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName,
bucketNum, columns, dataSchemaName, dataTableName, indexes, isImmutableRows, physicalNames,
- defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, stats, baseColumnCount, rowKeyOrderOptimizable);
+ defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, stats, baseColumnCount, rowKeyOrderOptimizable, isTransactional);
}
private PTableImpl(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state,
@@ -297,10 +300,10 @@ public class PTableImpl implements PTable {
PName parentSchemaName, PName parentTableName, List<PTable> indexes, boolean isImmutableRows,
List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant,
boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType,
- PTableStats stats, int baseColumnCount, boolean rowKeyOrderOptimizable) throws SQLException {
+ PTableStats stats, int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional) throws SQLException {
init(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns,
stats, schemaName, parentTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName,
- viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable);
+ viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, isTransactional);
}
@Override
@@ -328,7 +331,7 @@ public class PTableImpl implements PTable {
PName pkName, Integer bucketNum, List<PColumn> columns, PTableStats stats, PName parentSchemaName, PName parentTableName,
List<PTable> indexes, boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL,
boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId,
- IndexType indexType , int baseColumnCount, boolean rowKeyOrderOptimizable) throws SQLException {
+ IndexType indexType , int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional) throws SQLException {
Preconditions.checkNotNull(schemaName);
Preconditions.checkArgument(tenantId==null || tenantId.getBytes().length > 0); // tenantId should be null or not empty
int estimatedSize = SizedUtil.OBJECT_SIZE * 2 + 23 * SizedUtil.POINTER_SIZE + 4 * SizedUtil.INT_SIZE + 2 * SizedUtil.LONG_SIZE + 2 * SizedUtil.INT_OBJECT_SIZE +
@@ -357,6 +360,7 @@ public class PTableImpl implements PTable {
this.viewType = viewType;
this.viewIndexId = viewIndexId;
this.indexType = indexType;
+ this.isTransactional = isTransactional;
this.tableStats = stats;
this.rowKeyOrderOptimizable = rowKeyOrderOptimizable;
List<PColumn> pkColumns;
@@ -676,7 +680,7 @@ public class PTableImpl implements PTable {
private Put setValues;
private Delete unsetValues;
- private Delete deleteRow;
+ private Mutation deleteRow;
private final long ts;
public PRowImpl(KeyValueBuilder kvBuilder, ImmutableBytesWritable key, long ts, Integer bucketNum) {
@@ -713,7 +717,8 @@ public class PTableImpl implements PTable {
// way HBase works.
addQuietly(setValues, kvBuilder, kvBuilder.buildPut(keyPtr,
SchemaUtil.getEmptyColumnFamilyPtr(PTableImpl.this),
- QueryConstants.EMPTY_COLUMN_BYTES_PTR, ts, ByteUtil.EMPTY_BYTE_ARRAY_PTR));
+ QueryConstants.EMPTY_COLUMN_BYTES_PTR, ts,
+ QueryConstants.EMPTY_COLUMN_VALUE_BYTES_PTR));
mutations.add(setValues);
if (!unsetValues.isEmpty()) {
mutations.add(unsetValues);
@@ -779,11 +784,22 @@ public class PTableImpl implements PTable {
@Override
public void delete() {
newMutations();
- Delete delete = new Delete(key);
- for (PColumnFamily colFamily : families) {
- delete.addFamily(colFamily.getName().getBytes(), ts);
+ // we're using the Tephra column family delete marker here to prevent the translation
+ // of deletes to puts by the Tephra's TransactionProcessor
+ if (PTableImpl.this.isTransactional()) {
+ Put delete = new Put(key);
+ for (PColumnFamily colFamily : families) {
+ delete.add(colFamily.getName().getBytes(), TxConstants.FAMILY_DELETE_QUALIFIER, ts,
+ HConstants.EMPTY_BYTE_ARRAY);
+ }
+ deleteRow = delete;
+ } else {
+ Delete delete = new Delete(key);
+ for (PColumnFamily colFamily : families) {
+ delete.deleteFamily(colFamily.getName().getBytes(), ts);
+ }
+ deleteRow = delete;
}
- deleteRow = delete;
// No need to write to the WAL for indexes
if (PTableImpl.this.getType() == PTableType.INDEX) {
deleteRow.setDurability(Durability.SKIP_WAL);
@@ -888,8 +904,8 @@ public class PTableImpl implements PTable {
}
@Override
- public synchronized void getIndexMaintainers(ImmutableBytesWritable ptr, PhoenixConnection connection) {
- if (indexMaintainersPtr == null) {
+ public synchronized boolean getIndexMaintainers(ImmutableBytesWritable ptr, PhoenixConnection connection) {
+ if (indexMaintainersPtr == null || indexMaintainersPtr.getLength()==0) {
indexMaintainersPtr = new ImmutableBytesWritable();
if (indexes.isEmpty()) {
indexMaintainersPtr.set(ByteUtil.EMPTY_BYTE_ARRAY);
@@ -898,6 +914,7 @@ public class PTableImpl implements PTable {
}
}
ptr.set(indexMaintainersPtr.get(), indexMaintainersPtr.getOffset(), indexMaintainersPtr.getLength());
+ return indexMaintainersPtr.getLength() > 0;
}
@Override
@@ -1014,6 +1031,7 @@ public class PTableImpl implements PTable {
boolean disableWAL = table.getDisableWAL();
boolean multiTenant = table.getMultiTenant();
boolean storeNulls = table.getStoreNulls();
+ boolean isTransactional = table.getTransactional();
ViewType viewType = null;
String viewStatement = null;
List<PName> physicalNames = Collections.emptyList();
@@ -1044,7 +1062,7 @@ public class PTableImpl implements PTable {
result.init(tenantId, schemaName, tableName, tableType, indexState, timeStamp, sequenceNumber, pkName,
(bucketNum == NO_SALTING) ? null : bucketNum, columns, stats, schemaName,dataTableName, indexes,
isImmutableRows, physicalNames, defaultFamilyName, viewStatement, disableWAL,
- multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable);
+ multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, isTransactional);
return result;
} catch (SQLException e) {
throw new RuntimeException(e); // Impossible
@@ -1124,6 +1142,7 @@ public class PTableImpl implements PTable {
builder.setDisableWAL(table.isWALDisabled());
builder.setMultiTenant(table.isMultiTenant());
builder.setStoreNulls(table.getStoreNulls());
+ builder.setTransactional(table.isTransactional());
if(table.getType() == PTableType.VIEW){
builder.setViewType(ByteStringer.wrap(new byte[]{table.getViewType().getSerializedValue()}));
builder.setViewStatement(ByteStringer.wrap(PVarchar.INSTANCE.toBytes(table.getViewStatement())));
@@ -1155,6 +1174,10 @@ public class PTableImpl implements PTable {
}
@Override
+ public boolean isTransactional() {
+ return isTransactional;
+ }
+
public int getBaseColumnCount() {
return baseColumnCount;
}
[18/52] [abbrv] phoenix git commit: PHOENIX-2458
MutableIndexFailureIT causes other NeedsOwnMiniClusterTest tests to hang
Posted by ma...@apache.org.
PHOENIX-2458 MutableIndexFailureIT causes other NeedsOwnMiniClusterTest tests to hang
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b2676cbe
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b2676cbe
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b2676cbe
Branch: refs/heads/calcite
Commit: b2676cbe1e8ee08a69da6328ebc9aa93abc4499e
Parents: 3bfa38d
Author: Thomas D'Silva <td...@salesforce.com>
Authored: Sat Nov 28 20:09:43 2015 -0800
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Sat Nov 28 20:09:43 2015 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/index/MutableIndexFailureIT.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b2676cbe/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index 7c44401..801d6c3 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -95,6 +95,7 @@ import com.google.common.collect.Maps;
@Category(NeedsOwnMiniClusterTest.class)
@RunWith(Parameterized.class)
+@Ignore // see PHOENIX-2458
public class MutableIndexFailureIT extends BaseTest {
private static final int NUM_SLAVES = 4;
private static PhoenixTestDriver driver;
[37/52] [abbrv] phoenix git commit: PHOENIX-2480 SQL Query with
multiple projection selections over multiple tables having LEFT OUTER JOINS
returns completely null for random columns even when data is present
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/bc9974ba/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
new file mode 100644
index 0000000..d9016d0
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java
@@ -0,0 +1,314 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+public class SortMergeJoinMoreIT extends BaseHBaseManagedTimeIT {
+
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
+ // Forces server cache to be used
+ props.put(QueryServices.INDEX_MUTATE_BATCH_SIZE_THRESHOLD_ATTRIB, Integer.toString(2));
+ // Must update config before starting server
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @Test
+ public void testJoinOverSaltedTables() throws Exception {
+ String tempTableNoSalting = "TEMP_TABLE_NO_SALTING";
+ String tempTableWithSalting = "TEMP_TABLE_WITH_SALTING";
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ try {
+ conn.createStatement().execute("CREATE TABLE " + tempTableNoSalting
+ + " (mypk INTEGER NOT NULL PRIMARY KEY, "
+ + " col1 INTEGER)");
+ conn.createStatement().execute("CREATE TABLE " + tempTableWithSalting
+ + " (mypk INTEGER NOT NULL PRIMARY KEY, "
+ + " col1 INTEGER) SALT_BUCKETS=4");
+
+ PreparedStatement upsertStmt = conn.prepareStatement(
+ "upsert into " + tempTableNoSalting + "(mypk, col1) " + "values (?, ?)");
+ for (int i = 0; i < 3; i++) {
+ upsertStmt.setInt(1, i + 1);
+ upsertStmt.setInt(2, 3 - i);
+ upsertStmt.execute();
+ }
+ conn.commit();
+
+ upsertStmt = conn.prepareStatement(
+ "upsert into " + tempTableWithSalting + "(mypk, col1) " + "values (?, ?)");
+ for (int i = 0; i < 6; i++) {
+ upsertStmt.setInt(1, i + 1);
+ upsertStmt.setInt(2, 3 - (i % 3));
+ upsertStmt.execute();
+ }
+ conn.commit();
+
+ // LHS=unsalted JOIN RHS=salted
+ String query = "SELECT /*+ USE_SORT_MERGE_JOIN*/ lhs.mypk, lhs.col1, rhs.mypk, rhs.col1 FROM "
+ + tempTableNoSalting + " lhs JOIN "
+ + tempTableWithSalting + " rhs ON rhs.mypk = lhs.col1 ORDER BY lhs.mypk";
+ PreparedStatement statement = conn.prepareStatement(query);
+ ResultSet rs = statement.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 1);
+ assertEquals(rs.getInt(2), 3);
+ assertEquals(rs.getInt(3), 3);
+ assertEquals(rs.getInt(4), 1);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 2);
+ assertEquals(rs.getInt(2), 2);
+ assertEquals(rs.getInt(3), 2);
+ assertEquals(rs.getInt(4), 2);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 3);
+ assertEquals(rs.getInt(2), 1);
+ assertEquals(rs.getInt(3), 1);
+ assertEquals(rs.getInt(4), 3);
+
+ assertFalse(rs.next());
+
+ // LHS=salted JOIN RHS=salted
+ query = "SELECT /*+ USE_SORT_MERGE_JOIN*/ lhs.mypk, lhs.col1, rhs.mypk, rhs.col1 FROM "
+ + tempTableWithSalting + " lhs JOIN "
+ + tempTableNoSalting + " rhs ON rhs.mypk = lhs.col1 ORDER BY lhs.mypk";
+ statement = conn.prepareStatement(query);
+ rs = statement.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 1);
+ assertEquals(rs.getInt(2), 3);
+ assertEquals(rs.getInt(3), 3);
+ assertEquals(rs.getInt(4), 1);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 2);
+ assertEquals(rs.getInt(2), 2);
+ assertEquals(rs.getInt(3), 2);
+ assertEquals(rs.getInt(4), 2);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 3);
+ assertEquals(rs.getInt(2), 1);
+ assertEquals(rs.getInt(3), 1);
+ assertEquals(rs.getInt(4), 3);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 4);
+ assertEquals(rs.getInt(2), 3);
+ assertEquals(rs.getInt(3), 3);
+ assertEquals(rs.getInt(4), 1);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 5);
+ assertEquals(rs.getInt(2), 2);
+ assertEquals(rs.getInt(3), 2);
+ assertEquals(rs.getInt(4), 2);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 6);
+ assertEquals(rs.getInt(2), 1);
+ assertEquals(rs.getInt(3), 1);
+ assertEquals(rs.getInt(4), 3);
+
+ assertFalse(rs.next());
+
+ // LHS=salted JOIN RHS=salted
+ query = "SELECT /*+ USE_SORT_MERGE_JOIN*/ lhs.mypk, lhs.col1, rhs.mypk, rhs.col1 FROM "
+ + tempTableWithSalting + " lhs JOIN "
+ + tempTableWithSalting + " rhs ON rhs.mypk = (lhs.col1 + 3) ORDER BY lhs.mypk";
+ statement = conn.prepareStatement(query);
+ rs = statement.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 1);
+ assertEquals(rs.getInt(2), 3);
+ assertEquals(rs.getInt(3), 6);
+ assertEquals(rs.getInt(4), 1);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 2);
+ assertEquals(rs.getInt(2), 2);
+ assertEquals(rs.getInt(3), 5);
+ assertEquals(rs.getInt(4), 2);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 3);
+ assertEquals(rs.getInt(2), 1);
+ assertEquals(rs.getInt(3), 4);
+ assertEquals(rs.getInt(4), 3);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 4);
+ assertEquals(rs.getInt(2), 3);
+ assertEquals(rs.getInt(3), 6);
+ assertEquals(rs.getInt(4), 1);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 5);
+ assertEquals(rs.getInt(2), 2);
+ assertEquals(rs.getInt(3), 5);
+ assertEquals(rs.getInt(4), 2);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 6);
+ assertEquals(rs.getInt(2), 1);
+ assertEquals(rs.getInt(3), 4);
+ assertEquals(rs.getInt(4), 3);
+
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testJoinOnDynamicColumns() throws Exception {
+ String tableA = "tableA";
+ String tableB = "tableB";
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = null;
+ PreparedStatement stmt = null;
+ try {
+ conn = DriverManager.getConnection(getUrl(), props);
+ String ddlA = "CREATE TABLE " + tableA + " (pkA INTEGER NOT NULL, " + " colA1 INTEGER, "
+ + " colA2 VARCHAR " + "CONSTRAINT PK PRIMARY KEY" + "(pkA)" + ")";
+
+ String ddlB = "CREATE TABLE " + tableB + " (pkB INTEGER NOT NULL PRIMARY KEY, " + " colB INTEGER)";
+ stmt = conn.prepareStatement(ddlA);
+ stmt.execute();
+ stmt.close();
+
+ stmt = conn.prepareStatement(ddlB);
+ stmt.execute();
+ stmt.close();
+
+ String upsertA = "UPSERT INTO TABLEA (pkA, colA1, colA2) VALUES(?, ?, ?)";
+ stmt = conn.prepareStatement(upsertA);
+ int i = 0;
+ for (i = 0; i < 5; i++) {
+ stmt.setInt(1, i);
+ stmt.setInt(2, i + 10);
+ stmt.setString(3, "00" + i);
+ stmt.executeUpdate();
+ }
+ conn.commit();
+ stmt.close();
+
+ // upsert select dynamic columns in tableB
+ conn.createStatement().execute("CREATE SEQUENCE SEQB");
+ String upsertBSelectA = "UPSERT INTO TABLEB (pkB, pkA INTEGER)"
+ + "SELECT NEXT VALUE FOR SEQB, pkA FROM TABLEA";
+ stmt = conn.prepareStatement(upsertBSelectA);
+ stmt.executeUpdate();
+ stmt.close();
+ conn.commit();
+ conn.createStatement().execute("DROP SEQUENCE SEQB");
+
+ // perform a join between tableB and tableA by joining on the dynamic column that we upserted in
+ // tableB. This join should return all the rows from table A.
+ String joinSql = "SELECT /*+ USE_SORT_MERGE_JOIN*/ A.pkA, A.COLA1, A.colA2 FROM TABLEB B(pkA INTEGER) JOIN TABLEA A ON a.pkA = b.pkA";
+ stmt = conn.prepareStatement(joinSql);
+ ResultSet rs = stmt.executeQuery();
+ i = 0;
+ while (rs.next()) {
+ // check that we get back all the rows that we upserted for tableA above.
+ assertEquals(rs.getInt(1), i);
+ assertEquals(rs.getInt(2), i + 10);
+ assertEquals(rs.getString(3), "00" + i);
+ i++;
+ }
+ assertEquals(5,i);
+ } finally {
+ if (stmt != null) {
+ stmt.close();
+ }
+ if (conn != null) {
+ conn.close();
+ }
+
+ }
+
+ }
+
+ @Test
+ public void testSubqueryWithoutData() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+
+ try {
+ String GRAMMAR_TABLE = "CREATE TABLE IF NOT EXISTS GRAMMAR_TABLE (ID INTEGER PRIMARY KEY, " +
+ "unsig_id UNSIGNED_INT, big_id BIGINT, unsig_long_id UNSIGNED_LONG, tiny_id TINYINT," +
+ "unsig_tiny_id UNSIGNED_TINYINT, small_id SMALLINT, unsig_small_id UNSIGNED_SMALLINT," +
+ "float_id FLOAT, unsig_float_id UNSIGNED_FLOAT, double_id DOUBLE, unsig_double_id UNSIGNED_DOUBLE," +
+ "decimal_id DECIMAL, boolean_id BOOLEAN, time_id TIME, date_id DATE, timestamp_id TIMESTAMP," +
+ "unsig_time_id TIME, unsig_date_id DATE, unsig_timestamp_id TIMESTAMP, varchar_id VARCHAR (30)," +
+ "char_id CHAR (30), binary_id BINARY (100), varbinary_id VARBINARY (100))";
+
+ String LARGE_TABLE = "CREATE TABLE IF NOT EXISTS LARGE_TABLE (ID INTEGER PRIMARY KEY, " +
+ "unsig_id UNSIGNED_INT, big_id BIGINT, unsig_long_id UNSIGNED_LONG, tiny_id TINYINT," +
+ "unsig_tiny_id UNSIGNED_TINYINT, small_id SMALLINT, unsig_small_id UNSIGNED_SMALLINT," +
+ "float_id FLOAT, unsig_float_id UNSIGNED_FLOAT, double_id DOUBLE, unsig_double_id UNSIGNED_DOUBLE," +
+ "decimal_id DECIMAL, boolean_id BOOLEAN, time_id TIME, date_id DATE, timestamp_id TIMESTAMP," +
+ "unsig_time_id TIME, unsig_date_id DATE, unsig_timestamp_id TIMESTAMP, varchar_id VARCHAR (30)," +
+ "char_id CHAR (30), binary_id BINARY (100), varbinary_id VARBINARY (100))";
+
+ String SECONDARY_LARGE_TABLE = "CREATE TABLE IF NOT EXISTS SECONDARY_LARGE_TABLE (SEC_ID INTEGER PRIMARY KEY," +
+ "sec_unsig_id UNSIGNED_INT, sec_big_id BIGINT, sec_usnig_long_id UNSIGNED_LONG, sec_tiny_id TINYINT," +
+ "sec_unsig_tiny_id UNSIGNED_TINYINT, sec_small_id SMALLINT, sec_unsig_small_id UNSIGNED_SMALLINT," +
+ "sec_float_id FLOAT, sec_unsig_float_id UNSIGNED_FLOAT, sec_double_id DOUBLE, sec_unsig_double_id UNSIGNED_DOUBLE," +
+ "sec_decimal_id DECIMAL, sec_boolean_id BOOLEAN, sec_time_id TIME, sec_date_id DATE," +
+ "sec_timestamp_id TIMESTAMP, sec_unsig_time_id TIME, sec_unsig_date_id DATE, sec_unsig_timestamp_id TIMESTAMP," +
+ "sec_varchar_id VARCHAR (30), sec_char_id CHAR (30), sec_binary_id BINARY (100), sec_varbinary_id VARBINARY (100))";
+ createTestTable(getUrl(), GRAMMAR_TABLE);
+ createTestTable(getUrl(), LARGE_TABLE);
+ createTestTable(getUrl(), SECONDARY_LARGE_TABLE);
+
+ String ddl = "SELECT /*+USE_SORT_MERGE_JOIN*/ * FROM (SELECT ID, BIG_ID, DATE_ID FROM LARGE_TABLE AS A WHERE (A.ID % 5) = 0) AS A " +
+ "INNER JOIN (SELECT SEC_ID, SEC_TINY_ID, SEC_UNSIG_FLOAT_ID FROM SECONDARY_LARGE_TABLE AS B WHERE (B.SEC_ID % 5) = 0) AS B " +
+ "ON A.ID=B.SEC_ID WHERE A.DATE_ID > ALL (SELECT SEC_DATE_ID FROM SECONDARY_LARGE_TABLE LIMIT 100) " +
+ "AND B.SEC_UNSIG_FLOAT_ID = ANY (SELECT sec_unsig_float_id FROM SECONDARY_LARGE_TABLE " +
+ "WHERE SEC_ID > ALL (SELECT MIN (ID) FROM GRAMMAR_TABLE WHERE UNSIG_ID IS NULL) AND " +
+ "SEC_UNSIG_ID < ANY (SELECT DISTINCT(UNSIG_ID) FROM LARGE_TABLE WHERE UNSIG_ID<2500) LIMIT 1000) " +
+ "AND A.ID < 10000";
+ ResultSet rs = conn.createStatement().executeQuery(ddl);
+ assertFalse(rs.next());
+ } finally {
+ Statement statement = conn.createStatement();
+ String query = "drop table GRAMMAR_TABLE";
+ statement.executeUpdate(query);
+ query = "drop table LARGE_TABLE";
+ statement.executeUpdate(query);
+ query = "drop table SECONDARY_LARGE_TABLE";
+ statement.executeUpdate(query);
+ conn.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/bc9974ba/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
index 8a9abc2..ad65c1c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
@@ -285,23 +285,28 @@ public class QueryCompiler {
JoinType[] joinTypes = new JoinType[count];
PTable[] tables = new PTable[count];
int[] fieldPositions = new int[count];
- HashSubPlan[] subPlans = new HashSubPlan[count];
+ StatementContext[] subContexts = new StatementContext[count];
+ QueryPlan[] subPlans = new QueryPlan[count];
+ HashSubPlan[] hashPlans = new HashSubPlan[count];
fieldPositions[0] = projectedTable.getColumns().size() - projectedTable.getPKColumns().size();
for (int i = 0; i < count; i++) {
JoinSpec joinSpec = joinSpecs.get(i);
Scan subScan = ScanUtil.newScan(originalScan);
- StatementContext subContext = new StatementContext(statement, context.getResolver(), subScan, new SequenceManager(statement));
- QueryPlan joinPlan = compileJoinQuery(subContext, binds, joinSpec.getJoinTable(), true, true, null);
+ subContexts[i] = new StatementContext(statement, context.getResolver(), subScan, new SequenceManager(statement));
+ subPlans[i] = compileJoinQuery(subContexts[i], binds, joinSpec.getJoinTable(), true, true, null);
boolean hasPostReference = joinSpec.getJoinTable().hasPostReference();
if (hasPostReference) {
- tables[i] = subContext.getResolver().getTables().get(0).getTable();
+ tables[i] = subContexts[i].getResolver().getTables().get(0).getTable();
projectedTable = JoinCompiler.joinProjectedTables(projectedTable, tables[i], joinSpec.getType());
} else {
tables[i] = null;
}
+ }
+ for (int i = 0; i < count; i++) {
+ JoinSpec joinSpec = joinSpecs.get(i);
context.setResolver(FromCompiler.getResolverForProjectedTable(projectedTable, context.getConnection(), query.getUdfParseNodes()));
joinIds[i] = new ImmutableBytesPtr(emptyByteArray); // place-holder
- Pair<List<Expression>, List<Expression>> joinConditions = joinSpec.compileJoinConditions(context, subContext, true);
+ Pair<List<Expression>, List<Expression>> joinConditions = joinSpec.compileJoinConditions(context, subContexts[i], true);
joinExpressions[i] = joinConditions.getFirst();
List<Expression> hashExpressions = joinConditions.getSecond();
Pair<Expression, Expression> keyRangeExpressions = new Pair<Expression, Expression>(null, null);
@@ -312,7 +317,7 @@ public class QueryCompiler {
if (i < count - 1) {
fieldPositions[i + 1] = fieldPositions[i] + (tables[i] == null ? 0 : (tables[i].getColumns().size() - tables[i].getPKColumns().size()));
}
- subPlans[i] = new HashSubPlan(i, joinPlan, optimized ? null : hashExpressions, joinSpec.isSingleValueOnly(), keyRangeLhsExpression, keyRangeRhsExpression);
+ hashPlans[i] = new HashSubPlan(i, subPlans[i], optimized ? null : hashExpressions, joinSpec.isSingleValueOnly(), keyRangeLhsExpression, keyRangeRhsExpression);
}
TupleProjector.serializeProjectorIntoScan(context.getScan(), tupleProjector);
QueryPlan plan = compileSingleFlatQuery(context, query, binds, asSubquery, !asSubquery && joinTable.isAllLeftJoin(), null, !table.isSubselect() && projectPKColumns ? tupleProjector : null, true);
@@ -322,7 +327,7 @@ public class QueryCompiler {
limit = plan.getLimit();
}
HashJoinInfo joinInfo = new HashJoinInfo(projectedTable, joinIds, joinExpressions, joinTypes, starJoinVector, tables, fieldPositions, postJoinFilterExpression, limit);
- return HashJoinPlan.create(joinTable.getStatement(), plan, joinInfo, subPlans);
+ return HashJoinPlan.create(joinTable.getStatement(), plan, joinInfo, hashPlans);
}
JoinSpec lastJoinSpec = joinSpecs.get(joinSpecs.size() - 1);
[51/52] [abbrv] phoenix git commit: PHOENIX-2495 Fix test failures in
CsvBulkLoadToolIT
Posted by ma...@apache.org.
PHOENIX-2495 Fix test failures in CsvBulkLoadToolIT
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/f244feb4
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/f244feb4
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/f244feb4
Branch: refs/heads/calcite
Commit: f244feb4261d0edd19992f0d7f04a3a58615e479
Parents: 24a94a2
Author: Nick Dimiduk <nd...@apache.org>
Authored: Wed Dec 9 16:51:18 2015 -0800
Committer: Nick Dimiduk <nd...@apache.org>
Committed: Wed Dec 9 17:37:54 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java | 2 +-
.../org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java | 1 +
.../org/apache/phoenix/mapreduce/FormatToKeyValueMapper.java | 7 ++++++-
3 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f244feb4/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
index a5b7488..4a440d6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
@@ -228,7 +228,7 @@ public class CsvBulkLoadToolIT {
"--zookeeper", zkQuorum});
fail("Csv bulk load currently has issues with local indexes.");
} catch( UnsupportedOperationException ise) {
- assertEquals("Local indexes not supported by CSV Bulk Loader",ise.getMessage());
+ assertEquals("Local indexes not supported by Bulk Loader",ise.getMessage());
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f244feb4/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
index 5a5d378..2cb1ac7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
@@ -64,6 +64,7 @@ public class CsvToKeyValueMapper extends FormatToKeyValueMapper<CSVRecord> {
@Override
protected void setup(Context context) throws IOException, InterruptedException {
+ super.setup(context);
Configuration conf = context.getConfiguration();
lineParser = new CsvLineParser(
CsvBulkImportUtil.getCharacter(conf, FIELD_DELIMITER_CONFKEY),
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f244feb4/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueMapper.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueMapper.java
index b2e99e5..7e115e5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueMapper.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToKeyValueMapper.java
@@ -128,6 +128,9 @@ public abstract class FormatToKeyValueMapper<RECORD> extends Mapper<LongWritable
@SuppressWarnings("deprecation")
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
+ if (conn == null) {
+ throw new RuntimeException("Connection not initialized.");
+ }
try {
RECORD record = null;
try {
@@ -171,7 +174,9 @@ public abstract class FormatToKeyValueMapper<RECORD> extends Mapper<LongWritable
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
try {
- conn.close();
+ if (conn != null) {
+ conn.close();
+ }
} catch (SQLException e) {
throw new RuntimeException(e);
}
[43/52] [abbrv] phoenix git commit: PHOENIX-2490 Exception while
running update statistics on transactional table
Posted by ma...@apache.org.
PHOENIX-2490 Exception while running update statistics on transactional table
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8ce3b580
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8ce3b580
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8ce3b580
Branch: refs/heads/calcite
Commit: 8ce3b580fb14f23a49b3e2c47ab2628e140ae2da
Parents: 93acc27
Author: Thomas D'Silva <td...@salesforce.com>
Authored: Fri Dec 4 13:48:48 2015 -0800
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Mon Dec 7 12:22:13 2015 -0800
----------------------------------------------------------------------
.../phoenix/end2end/StatsCollectorIT.java | 33 ++++++++++++++------
.../apache/phoenix/compile/PostDDLCompiler.java | 3 +-
2 files changed, 25 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8ce3b580/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
index 6c392f5..23859d6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
@@ -59,27 +59,40 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
props.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
props.put(QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB, Boolean.TRUE.toString());
props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(1024));
+ props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
@Test
- public void testUpdateStatsForTheTable() throws Throwable {
- Connection conn;
+ public void testUpdateStatsForNonTxnTable() throws Throwable {
+ helpTestUpdateStats(false);
+ }
+
+ @Test
+ public void testUpdateStatsForTxnTable() throws Throwable {
+ helpTestUpdateStats(true);
+ }
+
+ private void helpTestUpdateStats(boolean transactional) throws SQLException, IOException,
+ InterruptedException {
+ Connection conn;
PreparedStatement stmt;
ResultSet rs;
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String tableName = "T" + (transactional ? "_TXN" : "");
// props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
conn = DriverManager.getConnection(getUrl(), props);
conn.createStatement().execute(
- "CREATE TABLE t ( k VARCHAR, a_string_array VARCHAR(100) ARRAY[4], b_string_array VARCHAR(100) ARRAY[4] \n"
- + " CONSTRAINT pk PRIMARY KEY (k, b_string_array DESC)) \n");
+ "CREATE TABLE " + tableName +" ( k VARCHAR, a_string_array VARCHAR(100) ARRAY[4], b_string_array VARCHAR(100) ARRAY[4] \n"
+ + " CONSTRAINT pk PRIMARY KEY (k, b_string_array DESC))"
+ + (transactional ? " TRANSACTIONAL=true" : ""));
String[] s;
Array array;
- conn = upsertValues(props, "t");
+ conn = upsertValues(props, tableName);
// CAll the update statistics query here. If already major compaction has run this will not get executed.
- stmt = conn.prepareStatement("UPDATE STATISTICS T");
+ stmt = conn.prepareStatement("UPDATE STATISTICS " + tableName);
stmt.execute();
- stmt = upsertStmt(conn, "t");
+ stmt = upsertStmt(conn, tableName);
stmt.setString(1, "z");
s = new String[] { "xyz", "def", "ghi", "jkll", null, null, "xxx" };
array = conn.createArrayOf("VARCHAR", s);
@@ -91,12 +104,12 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
conn.close();
conn = DriverManager.getConnection(getUrl(), props);
// This analyze would not work
- stmt = conn.prepareStatement("UPDATE STATISTICS T");
+ stmt = conn.prepareStatement("UPDATE STATISTICS " + tableName);
stmt.execute();
- rs = conn.createStatement().executeQuery("SELECT k FROM T");
+ rs = conn.createStatement().executeQuery("SELECT k FROM " + tableName);
assertTrue(rs.next());
conn.close();
- }
+ }
@Test
public void testUpdateStatsWithMultipleTables() throws Throwable {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8ce3b580/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
index 506623b..192a031 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
@@ -21,6 +21,7 @@ import java.sql.SQLException;
import java.util.Collections;
import java.util.List;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.cache.ServerCacheClient.ServerCache;
@@ -184,7 +185,7 @@ public class PostDDLCompiler {
// FIXME: DDL operations aren't transactional, so we're basing the timestamp on a server timestamp.
// Not sure what the fix should be. We don't need conflict detection nor filtering of invalid transactions
// in this case, so maybe this is ok.
- if (tableRef.getTable().isTransactional()) {
+ if (ts!=HConstants.LATEST_TIMESTAMP && tableRef.getTable().isTransactional()) {
ts = TransactionUtil.convertToNanoseconds(ts);
}
ScanUtil.setTimeRange(scan, ts);
[38/52] [abbrv] phoenix git commit: PHOENIX-2480 SQL Query with
multiple projection selections over multiple tables having LEFT OUTER JOINS
returns completely null for random columns even when data is present
Posted by ma...@apache.org.
PHOENIX-2480 SQL Query with multiple projection selections over multiple tables having LEFT OUTER JOINS returns completely null for random columns even when data is present
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/bc9974ba
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/bc9974ba
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/bc9974ba
Branch: refs/heads/calcite
Commit: bc9974ba1c4cf3fd7b3cba9f4120b0a4b4c81610
Parents: 30ef0cc
Author: maryannxue <we...@intel.com>
Authored: Wed Dec 2 21:11:44 2015 -0500
Committer: maryannxue <we...@intel.com>
Committed: Wed Dec 2 21:11:44 2015 -0500
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/HashJoinIT.java | 608 +----------------
.../apache/phoenix/end2end/HashJoinMoreIT.java | 678 +++++++++++++++++++
.../apache/phoenix/end2end/SortMergeJoinIT.java | 262 -------
.../phoenix/end2end/SortMergeJoinMoreIT.java | 314 +++++++++
.../apache/phoenix/compile/QueryCompiler.java | 19 +-
5 files changed, 1005 insertions(+), 876 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/bc9974ba/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
index 433f74f..3d8b006 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
@@ -34,7 +34,6 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.sql.Array;
import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
@@ -416,53 +415,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
" DYNAMIC SERVER FILTER BY \"S.supplier_id\" IN (\"I.supplier_id\")\n" +
" JOIN-SCANNER 4 ROW LIMIT",
/*
- * testJoinWithKeyRangeOptimization()
- * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
- * FROM TEMP_TABLE_COMPOSITE_PK lhs
- * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col1 = rhs.col2
- */
- "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- "CLIENT MERGE SORT\n" +
- " PARALLEL INNER-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- " CLIENT MERGE SORT",
- /*
- * testJoinWithKeyRangeOptimization()
- * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
- * FROM TEMP_TABLE_COMPOSITE_PK lhs
- * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col0 = rhs.col2
- */
- "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- "CLIENT MERGE SORT\n" +
- " PARALLEL INNER-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- " CLIENT MERGE SORT\n" +
- " DYNAMIC SERVER FILTER BY LHS.COL0 IN (RHS.COL2)",
- /*
- * testJoinWithKeyRangeOptimization()
- * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
- * FROM TEMP_TABLE_COMPOSITE_PK lhs
- * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col0 = rhs.col1 AND lhs.col1 = rhs.col2
- */
- "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- "CLIENT MERGE SORT\n" +
- " PARALLEL INNER-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- " CLIENT MERGE SORT\n" +
- " DYNAMIC SERVER FILTER BY (LHS.COL0, LHS.COL1) IN ((RHS.COL1, RHS.COL2))",
- /*
- * testJoinWithKeyRangeOptimization()
- * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
- * FROM TEMP_TABLE_COMPOSITE_PK lhs
- * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col0 = rhs.col1 AND lhs.col2 = rhs.col3 - 1 AND lhs.col1 = rhs.col2
- */
- "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- "CLIENT MERGE SORT\n" +
- " PARALLEL INNER-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- " CLIENT MERGE SORT\n" +
- " DYNAMIC SERVER FILTER BY (LHS.COL0, LHS.COL1, LHS.COL2) IN ((RHS.COL1, RHS.COL2, TO_INTEGER((RHS.COL3 - 1))))",
- /*
* testJoinWithSetMaxRows()
* statement.setMaxRows(4);
* SELECT order_id, i.name, quantity FROM joinItemTable i
@@ -795,53 +747,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
" DYNAMIC SERVER FILTER BY \"S.supplier_id\" IN (\"I.0:supplier_id\")\n" +
" JOIN-SCANNER 4 ROW LIMIT",
/*
- * testJoinWithKeyRangeOptimization()
- * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
- * FROM TEMP_TABLE_COMPOSITE_PK lhs
- * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col1 = rhs.col2
- */
- "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- "CLIENT MERGE SORT\n" +
- " PARALLEL INNER-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- " CLIENT MERGE SORT",
- /*
- * testJoinWithKeyRangeOptimization()
- * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
- * FROM TEMP_TABLE_COMPOSITE_PK lhs
- * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col0 = rhs.col2
- */
- "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- "CLIENT MERGE SORT\n" +
- " PARALLEL INNER-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- " CLIENT MERGE SORT\n" +
- " DYNAMIC SERVER FILTER BY LHS.COL0 IN (RHS.COL2)",
- /*
- * testJoinWithKeyRangeOptimization()
- * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
- * FROM TEMP_TABLE_COMPOSITE_PK lhs
- * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col0 = rhs.col1 AND lhs.col1 = rhs.col2
- */
- "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- "CLIENT MERGE SORT\n" +
- " PARALLEL INNER-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- " CLIENT MERGE SORT\n" +
- " DYNAMIC SERVER FILTER BY (LHS.COL0, LHS.COL1) IN ((RHS.COL1, RHS.COL2))",
- /*
- * testJoinWithKeyRangeOptimization()
- * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
- * FROM TEMP_TABLE_COMPOSITE_PK lhs
- * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col0 = rhs.col1 AND lhs.col2 = rhs.col3 - 1 AND lhs.col1 = rhs.col2
- */
- "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- "CLIENT MERGE SORT\n" +
- " PARALLEL INNER-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- " CLIENT MERGE SORT\n" +
- " DYNAMIC SERVER FILTER BY (LHS.COL0, LHS.COL1, LHS.COL2) IN ((RHS.COL1, RHS.COL2, TO_INTEGER((RHS.COL3 - 1))))",
- /*
* testJoinWithSetMaxRows()
* statement.setMaxRows(4);
* SELECT order_id, i.name, quantity FROM joinItemTable i
@@ -1196,53 +1101,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
" DYNAMIC SERVER FILTER BY \"S.supplier_id\" IN (\"I.0:supplier_id\")\n" +
" JOIN-SCANNER 4 ROW LIMIT",
/*
- * testJoinWithKeyRangeOptimization()
- * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
- * FROM TEMP_TABLE_COMPOSITE_PK lhs
- * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col1 = rhs.col2
- */
- "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- "CLIENT MERGE SORT\n" +
- " PARALLEL INNER-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- " CLIENT MERGE SORT",
- /*
- * testJoinWithKeyRangeOptimization()
- * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
- * FROM TEMP_TABLE_COMPOSITE_PK lhs
- * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col0 = rhs.col2
- */
- "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- "CLIENT MERGE SORT\n" +
- " PARALLEL INNER-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- " CLIENT MERGE SORT\n" +
- " DYNAMIC SERVER FILTER BY LHS.COL0 IN (RHS.COL2)",
- /*
- * testJoinWithKeyRangeOptimization()
- * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
- * FROM TEMP_TABLE_COMPOSITE_PK lhs
- * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col0 = rhs.col1 AND lhs.col1 = rhs.col2
- */
- "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- "CLIENT MERGE SORT\n" +
- " PARALLEL INNER-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- " CLIENT MERGE SORT\n" +
- " DYNAMIC SERVER FILTER BY (LHS.COL0, LHS.COL1) IN ((RHS.COL1, RHS.COL2))",
- /*
- * testJoinWithKeyRangeOptimization()
- * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
- * FROM TEMP_TABLE_COMPOSITE_PK lhs
- * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col0 = rhs.col1 AND lhs.col2 = rhs.col3 - 1 AND lhs.col1 = rhs.col2
- */
- "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- "CLIENT MERGE SORT\n" +
- " PARALLEL INNER-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
- " CLIENT MERGE SORT\n" +
- " DYNAMIC SERVER FILTER BY (LHS.COL0, LHS.COL1, LHS.COL2) IN ((RHS.COL1, RHS.COL2, TO_INTEGER((RHS.COL3 - 1))))",
- /*
* testJoinWithSetMaxRows()
* statement.setMaxRows(4);
* SELECT order_id, i.name, quantity FROM joinItemTable i
@@ -2843,213 +2701,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
}
@Test
- public void testJoinOverSaltedTables() throws Exception {
- String tempTableNoSalting = "TEMP_TABLE_NO_SALTING";
- String tempTableWithSalting = "TEMP_TABLE_WITH_SALTING";
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- try {
- conn.createStatement().execute("CREATE TABLE " + tempTableNoSalting
- + " (mypk INTEGER NOT NULL PRIMARY KEY, "
- + " col1 INTEGER)");
- conn.createStatement().execute("CREATE TABLE " + tempTableWithSalting
- + " (mypk INTEGER NOT NULL PRIMARY KEY, "
- + " col1 INTEGER) SALT_BUCKETS=4");
-
- PreparedStatement upsertStmt = conn.prepareStatement(
- "upsert into " + tempTableNoSalting + "(mypk, col1) " + "values (?, ?)");
- for (int i = 0; i < 3; i++) {
- upsertStmt.setInt(1, i + 1);
- upsertStmt.setInt(2, 3 - i);
- upsertStmt.execute();
- }
- conn.commit();
-
- upsertStmt = conn.prepareStatement(
- "upsert into " + tempTableWithSalting + "(mypk, col1) " + "values (?, ?)");
- for (int i = 0; i < 6; i++) {
- upsertStmt.setInt(1, i + 1);
- upsertStmt.setInt(2, 3 - (i % 3));
- upsertStmt.execute();
- }
- conn.commit();
-
- // LHS=unsalted JOIN RHS=salted
- String query = "SELECT lhs.mypk, lhs.col1, rhs.mypk, rhs.col1 FROM "
- + tempTableNoSalting + " lhs JOIN "
- + tempTableWithSalting + " rhs ON rhs.mypk = lhs.col1";
- PreparedStatement statement = conn.prepareStatement(query);
- ResultSet rs = statement.executeQuery();
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 1);
- assertEquals(rs.getInt(2), 3);
- assertEquals(rs.getInt(3), 3);
- assertEquals(rs.getInt(4), 1);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 2);
- assertEquals(rs.getInt(2), 2);
- assertEquals(rs.getInt(3), 2);
- assertEquals(rs.getInt(4), 2);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 3);
- assertEquals(rs.getInt(2), 1);
- assertEquals(rs.getInt(3), 1);
- assertEquals(rs.getInt(4), 3);
-
- assertFalse(rs.next());
-
- // LHS=salted JOIN RHS=salted
- query = "SELECT lhs.mypk, lhs.col1, rhs.mypk, rhs.col1 FROM "
- + tempTableWithSalting + " lhs JOIN "
- + tempTableNoSalting + " rhs ON rhs.mypk = lhs.col1";
- statement = conn.prepareStatement(query);
- rs = statement.executeQuery();
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 1);
- assertEquals(rs.getInt(2), 3);
- assertEquals(rs.getInt(3), 3);
- assertEquals(rs.getInt(4), 1);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 2);
- assertEquals(rs.getInt(2), 2);
- assertEquals(rs.getInt(3), 2);
- assertEquals(rs.getInt(4), 2);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 3);
- assertEquals(rs.getInt(2), 1);
- assertEquals(rs.getInt(3), 1);
- assertEquals(rs.getInt(4), 3);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 4);
- assertEquals(rs.getInt(2), 3);
- assertEquals(rs.getInt(3), 3);
- assertEquals(rs.getInt(4), 1);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 5);
- assertEquals(rs.getInt(2), 2);
- assertEquals(rs.getInt(3), 2);
- assertEquals(rs.getInt(4), 2);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 6);
- assertEquals(rs.getInt(2), 1);
- assertEquals(rs.getInt(3), 1);
- assertEquals(rs.getInt(4), 3);
-
- assertFalse(rs.next());
-
- // LHS=salted JOIN RHS=salted
- query = "SELECT lhs.mypk, lhs.col1, rhs.mypk, rhs.col1 FROM "
- + tempTableWithSalting + " lhs JOIN "
- + tempTableWithSalting + " rhs ON rhs.mypk = (lhs.col1 + 3)";
- statement = conn.prepareStatement(query);
- rs = statement.executeQuery();
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 1);
- assertEquals(rs.getInt(2), 3);
- assertEquals(rs.getInt(3), 6);
- assertEquals(rs.getInt(4), 1);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 2);
- assertEquals(rs.getInt(2), 2);
- assertEquals(rs.getInt(3), 5);
- assertEquals(rs.getInt(4), 2);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 3);
- assertEquals(rs.getInt(2), 1);
- assertEquals(rs.getInt(3), 4);
- assertEquals(rs.getInt(4), 3);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 4);
- assertEquals(rs.getInt(2), 3);
- assertEquals(rs.getInt(3), 6);
- assertEquals(rs.getInt(4), 1);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 5);
- assertEquals(rs.getInt(2), 2);
- assertEquals(rs.getInt(3), 5);
- assertEquals(rs.getInt(4), 2);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 6);
- assertEquals(rs.getInt(2), 1);
- assertEquals(rs.getInt(3), 4);
- assertEquals(rs.getInt(4), 3);
-
- assertFalse(rs.next());
- } finally {
- conn.close();
- }
- }
-
- @Test
- public void testJoinOnDynamicColumns() throws Exception {
- String tableA = "tableA";
- String tableB = "tableB";
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = null;
- PreparedStatement stmt = null;
- try {
- conn = DriverManager.getConnection(getUrl(), props);
- String ddlA = "CREATE TABLE " + tableA + " (pkA INTEGER NOT NULL, " + " colA1 INTEGER, "
- + " colA2 VARCHAR " + "CONSTRAINT PK PRIMARY KEY" + "(pkA)" + ")";
-
- String ddlB = "CREATE TABLE " + tableB + " (pkB INTEGER NOT NULL PRIMARY KEY, " + " colB INTEGER)";
- stmt = conn.prepareStatement(ddlA);
- stmt.execute();
- stmt.close();
-
- stmt = conn.prepareStatement(ddlB);
- stmt.execute();
- stmt.close();
-
- String upsertA = "UPSERT INTO TABLEA (pkA, colA1, colA2) VALUES(?, ?, ?)";
- stmt = conn.prepareStatement(upsertA);
- int i = 0;
- for (i = 0; i < 5; i++) {
- stmt.setInt(1, i);
- stmt.setInt(2, i + 10);
- stmt.setString(3, "00" + i);
- stmt.executeUpdate();
- }
- conn.commit();
- stmt.close();
-
- // upsert select dynamic columns in tableB
- conn.createStatement().execute("CREATE SEQUENCE SEQB");
- String upsertBSelectA = "UPSERT INTO TABLEB (pkB, pkA INTEGER)"
- + "SELECT NEXT VALUE FOR SEQB, pkA FROM TABLEA";
- stmt = conn.prepareStatement(upsertBSelectA);
- stmt.executeUpdate();
- stmt.close();
- conn.commit();
- conn.createStatement().execute("DROP SEQUENCE SEQB");
-
- // perform a join between tableB and tableA by joining on the dynamic column that we upserted in
- // tableB. This join should return all the rows from table A.
- String joinSql = "SELECT A.pkA, A.COLA1, A.colA2 FROM TABLEB B(pkA INTEGER) JOIN TABLEA A ON a.pkA = b.pkA";
- stmt = conn.prepareStatement(joinSql);
- ResultSet rs = stmt.executeQuery();
- i = 0;
- while (rs.next()) {
- // check that we get back all the rows that we upserted for tableA above.
- assertEquals(rs.getInt(1), i);
- assertEquals(rs.getInt(2), i + 10);
- assertEquals(rs.getString(3), "00" + i);
- i++;
- }
- assertEquals(5,i);
- } finally {
- if (stmt != null) {
- stmt.close();
- }
- if (conn != null) {
- conn.close();
- }
-
- }
-
- }
-
- @Test
public void testSubJoin() throws Exception {
String query1 = "SELECT i.name, count(c.name), min(s.name), max(quantity) FROM " + JOIN_ORDER_TABLE_FULL_NAME + " o LEFT JOIN "
+ "(" + JOIN_SUPPLIER_TABLE_FULL_NAME + " s RIGHT JOIN " + JOIN_ITEM_TABLE_FULL_NAME + " i ON i.\"supplier_id\" = s.\"supplier_id\")"
@@ -3574,146 +3225,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
conn.close();
}
}
-
- @Test
- public void testJoinWithKeyRangeOptimization() throws Exception {
- String tempTableWithCompositePK = "TEMP_TABLE_COMPOSITE_PK";
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- try {
- conn.createStatement().execute("CREATE TABLE " + tempTableWithCompositePK
- + " (col0 INTEGER NOT NULL, "
- + " col1 INTEGER NOT NULL, "
- + " col2 INTEGER NOT NULL, "
- + " col3 INTEGER "
- + " CONSTRAINT pk PRIMARY KEY (col0, col1, col2)) "
- + " SALT_BUCKETS=4");
-
- PreparedStatement upsertStmt = conn.prepareStatement(
- "upsert into " + tempTableWithCompositePK + "(col0, col1, col2, col3) " + "values (?, ?, ?, ?)");
- for (int i = 0; i < 3; i++) {
- upsertStmt.setInt(1, i + 1);
- upsertStmt.setInt(2, i + 2);
- upsertStmt.setInt(3, i + 3);
- upsertStmt.setInt(4, i + 5);
- upsertStmt.execute();
- }
- conn.commit();
-
- // No leading part of PK
- String query = "SELECT lhs.col0, lhs.col1, lhs.col2, lhs.col3, rhs.col0, rhs.col1, rhs.col2, rhs.col3 FROM "
- + tempTableWithCompositePK + " lhs JOIN "
- + tempTableWithCompositePK + " rhs ON lhs.col1 = rhs.col2";
- PreparedStatement statement = conn.prepareStatement(query);
- ResultSet rs = statement.executeQuery();
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 2);
- assertEquals(rs.getInt(2), 3);
- assertEquals(rs.getInt(3), 4);
- assertEquals(rs.getInt(4), 6);
- assertEquals(rs.getInt(5), 1);
- assertEquals(rs.getInt(6), 2);
- assertEquals(rs.getInt(7), 3);
- assertEquals(rs.getInt(8), 5);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 3);
- assertEquals(rs.getInt(2), 4);
- assertEquals(rs.getInt(3), 5);
- assertEquals(rs.getInt(4), 7);
- assertEquals(rs.getInt(5), 2);
- assertEquals(rs.getInt(6), 3);
- assertEquals(rs.getInt(7), 4);
- assertEquals(rs.getInt(8), 6);
-
- assertFalse(rs.next());
-
- rs = conn.createStatement().executeQuery("EXPLAIN " + query);
- assertEquals(plans[21], QueryUtil.getExplainPlan(rs));
-
- // Two parts of PK but only one leading part
- query = "SELECT lhs.col0, lhs.col1, lhs.col2, lhs.col3, rhs.col0, rhs.col1, rhs.col2, rhs.col3 FROM "
- + tempTableWithCompositePK + " lhs JOIN "
- + tempTableWithCompositePK + " rhs ON lhs.col2 = rhs.col3 AND lhs.col0 = rhs.col2";
- statement = conn.prepareStatement(query);
- rs = statement.executeQuery();
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 3);
- assertEquals(rs.getInt(2), 4);
- assertEquals(rs.getInt(3), 5);
- assertEquals(rs.getInt(4), 7);
- assertEquals(rs.getInt(5), 1);
- assertEquals(rs.getInt(6), 2);
- assertEquals(rs.getInt(7), 3);
- assertEquals(rs.getInt(8), 5);
-
- assertFalse(rs.next());
-
- rs = conn.createStatement().executeQuery("EXPLAIN " + query);
- assertEquals(plans[22], QueryUtil.getExplainPlan(rs));
-
- // Two leading parts of PK
- query = "SELECT lhs.col0, lhs.col1, lhs.col2, lhs.col3, rhs.col0, rhs.col1, rhs.col2, rhs.col3 FROM "
- + tempTableWithCompositePK + " lhs JOIN "
- + tempTableWithCompositePK + " rhs ON lhs.col1 = rhs.col2 AND lhs.col0 = rhs.col1";
- statement = conn.prepareStatement(query);
- rs = statement.executeQuery();
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 2);
- assertEquals(rs.getInt(2), 3);
- assertEquals(rs.getInt(3), 4);
- assertEquals(rs.getInt(4), 6);
- assertEquals(rs.getInt(5), 1);
- assertEquals(rs.getInt(6), 2);
- assertEquals(rs.getInt(7), 3);
- assertEquals(rs.getInt(8), 5);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 3);
- assertEquals(rs.getInt(2), 4);
- assertEquals(rs.getInt(3), 5);
- assertEquals(rs.getInt(4), 7);
- assertEquals(rs.getInt(5), 2);
- assertEquals(rs.getInt(6), 3);
- assertEquals(rs.getInt(7), 4);
- assertEquals(rs.getInt(8), 6);
-
- assertFalse(rs.next());
-
- rs = conn.createStatement().executeQuery("EXPLAIN " + query);
- assertEquals(plans[23], QueryUtil.getExplainPlan(rs));
-
- // All parts of PK
- query = "SELECT lhs.col0, lhs.col1, lhs.col2, lhs.col3, rhs.col0, rhs.col1, rhs.col2, rhs.col3 FROM "
- + tempTableWithCompositePK + " lhs JOIN "
- + tempTableWithCompositePK + " rhs ON lhs.col1 = rhs.col2 AND lhs.col2 = rhs.col3 - 1 AND lhs.col0 = rhs.col1";
- statement = conn.prepareStatement(query);
- rs = statement.executeQuery();
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 2);
- assertEquals(rs.getInt(2), 3);
- assertEquals(rs.getInt(3), 4);
- assertEquals(rs.getInt(4), 6);
- assertEquals(rs.getInt(5), 1);
- assertEquals(rs.getInt(6), 2);
- assertEquals(rs.getInt(7), 3);
- assertEquals(rs.getInt(8), 5);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 3);
- assertEquals(rs.getInt(2), 4);
- assertEquals(rs.getInt(3), 5);
- assertEquals(rs.getInt(4), 7);
- assertEquals(rs.getInt(5), 2);
- assertEquals(rs.getInt(6), 3);
- assertEquals(rs.getInt(7), 4);
- assertEquals(rs.getInt(8), 6);
-
- assertFalse(rs.next());
-
- rs = conn.createStatement().executeQuery("EXPLAIN " + query);
- assertEquals(plans[24], QueryUtil.getExplainPlan(rs));
- } finally {
- conn.close();
- }
- }
@Test
public void testNonEquiJoin() throws Exception {
@@ -3807,129 +3318,12 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
rs = statement.executeQuery("EXPLAIN " + query);
- assertEquals(plans[25], QueryUtil.getExplainPlan(rs));
+ assertEquals(plans[21], QueryUtil.getExplainPlan(rs));
}
} finally {
conn.close();
}
}
-
- @Test
- public void testSubqueryWithoutData() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- conn.setAutoCommit(false);
-
- try {
- String GRAMMAR_TABLE = "CREATE TABLE IF NOT EXISTS GRAMMAR_TABLE (ID INTEGER PRIMARY KEY, " +
- "unsig_id UNSIGNED_INT, big_id BIGINT, unsig_long_id UNSIGNED_LONG, tiny_id TINYINT," +
- "unsig_tiny_id UNSIGNED_TINYINT, small_id SMALLINT, unsig_small_id UNSIGNED_SMALLINT," +
- "float_id FLOAT, unsig_float_id UNSIGNED_FLOAT, double_id DOUBLE, unsig_double_id UNSIGNED_DOUBLE," +
- "decimal_id DECIMAL, boolean_id BOOLEAN, time_id TIME, date_id DATE, timestamp_id TIMESTAMP," +
- "unsig_time_id TIME, unsig_date_id DATE, unsig_timestamp_id TIMESTAMP, varchar_id VARCHAR (30)," +
- "char_id CHAR (30), binary_id BINARY (100), varbinary_id VARBINARY (100))";
-
- String LARGE_TABLE = "CREATE TABLE IF NOT EXISTS LARGE_TABLE (ID INTEGER PRIMARY KEY, " +
- "unsig_id UNSIGNED_INT, big_id BIGINT, unsig_long_id UNSIGNED_LONG, tiny_id TINYINT," +
- "unsig_tiny_id UNSIGNED_TINYINT, small_id SMALLINT, unsig_small_id UNSIGNED_SMALLINT," +
- "float_id FLOAT, unsig_float_id UNSIGNED_FLOAT, double_id DOUBLE, unsig_double_id UNSIGNED_DOUBLE," +
- "decimal_id DECIMAL, boolean_id BOOLEAN, time_id TIME, date_id DATE, timestamp_id TIMESTAMP," +
- "unsig_time_id TIME, unsig_date_id DATE, unsig_timestamp_id TIMESTAMP, varchar_id VARCHAR (30)," +
- "char_id CHAR (30), binary_id BINARY (100), varbinary_id VARBINARY (100))";
-
- String SECONDARY_LARGE_TABLE = "CREATE TABLE IF NOT EXISTS SECONDARY_LARGE_TABLE (SEC_ID INTEGER PRIMARY KEY," +
- "sec_unsig_id UNSIGNED_INT, sec_big_id BIGINT, sec_usnig_long_id UNSIGNED_LONG, sec_tiny_id TINYINT," +
- "sec_unsig_tiny_id UNSIGNED_TINYINT, sec_small_id SMALLINT, sec_unsig_small_id UNSIGNED_SMALLINT," +
- "sec_float_id FLOAT, sec_unsig_float_id UNSIGNED_FLOAT, sec_double_id DOUBLE, sec_unsig_double_id UNSIGNED_DOUBLE," +
- "sec_decimal_id DECIMAL, sec_boolean_id BOOLEAN, sec_time_id TIME, sec_date_id DATE," +
- "sec_timestamp_id TIMESTAMP, sec_unsig_time_id TIME, sec_unsig_date_id DATE, sec_unsig_timestamp_id TIMESTAMP," +
- "sec_varchar_id VARCHAR (30), sec_char_id CHAR (30), sec_binary_id BINARY (100), sec_varbinary_id VARBINARY (100))";
- createTestTable(getUrl(), GRAMMAR_TABLE);
- createTestTable(getUrl(), LARGE_TABLE);
- createTestTable(getUrl(), SECONDARY_LARGE_TABLE);
-
- String ddl = "SELECT * FROM (SELECT ID, BIG_ID, DATE_ID FROM LARGE_TABLE AS A WHERE (A.ID % 5) = 0) AS A " +
- "INNER JOIN (SELECT SEC_ID, SEC_TINY_ID, SEC_UNSIG_FLOAT_ID FROM SECONDARY_LARGE_TABLE AS B WHERE (B.SEC_ID % 5) = 0) AS B " +
- "ON A.ID=B.SEC_ID WHERE A.DATE_ID > ALL (SELECT SEC_DATE_ID FROM SECONDARY_LARGE_TABLE LIMIT 100) " +
- "AND B.SEC_UNSIG_FLOAT_ID = ANY (SELECT sec_unsig_float_id FROM SECONDARY_LARGE_TABLE " +
- "WHERE SEC_ID > ALL (SELECT MIN (ID) FROM GRAMMAR_TABLE WHERE UNSIG_ID IS NULL) AND " +
- "SEC_UNSIG_ID < ANY (SELECT DISTINCT(UNSIG_ID) FROM LARGE_TABLE WHERE UNSIG_ID<2500) LIMIT 1000) " +
- "AND A.ID < 10000";
- ResultSet rs = conn.createStatement().executeQuery(ddl);
- assertFalse(rs.next());
- } finally {
- Statement statement = conn.createStatement();
- String query = "drop table GRAMMAR_TABLE";
- statement.executeUpdate(query);
- query = "drop table LARGE_TABLE";
- statement.executeUpdate(query);
- query = "drop table SECONDARY_LARGE_TABLE";
- statement.executeUpdate(query);
- conn.close();
- }
- }
-
- // PHOENIX-2381
- @Test
- public void testJoinWithMultiTenancy() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- try {
- conn.createStatement().execute("CREATE TABLE INVENTORY (" +
- " TENANTID UNSIGNED_INT NOT NULL" +
- ",ID UNSIGNED_INT NOT NULL" +
- ",FOO UNSIGNED_INT NOT NULL" +
- ",TIMESTAMP UNSIGNED_LONG NOT NULL" +
- ",CODES INTEGER ARRAY[] NOT NULL" +
- ",V UNSIGNED_LONG" +
- " CONSTRAINT pk PRIMARY KEY (TENANTID, ID, FOO, TIMESTAMP, CODES))" +
- " DEFAULT_COLUMN_FAMILY ='E'," +
- " MULTI_TENANT=true");
- PreparedStatement upsertStmt = conn.prepareStatement(
- "upsert into INVENTORY "
- + "(tenantid, id, foo, timestamp, codes) "
- + "values (?, ?, ?, ?, ?)");
- upsertStmt.setInt(1, 15);
- upsertStmt.setInt(2, 5);
- upsertStmt.setInt(3, 0);
- upsertStmt.setLong(4, 6);
- Array array = conn.createArrayOf("INTEGER", new Object[] {1, 2});
- upsertStmt.setArray(5, array);
- upsertStmt.executeUpdate();
- conn.commit();
-
- conn.createStatement().execute("CREATE TABLE PRODUCT_IDS (" +
- " PRODUCT_ID UNSIGNED_INT NOT NULL" +
- ",PRODUCT_NAME VARCHAR" +
- " CONSTRAINT pk PRIMARY KEY (PRODUCT_ID))" +
- " DEFAULT_COLUMN_FAMILY ='NAME'");
- upsertStmt = conn.prepareStatement(
- "upsert into PRODUCT_IDS "
- + "(product_id, product_name) "
- + "values (?, ?)");
- upsertStmt.setInt(1, 5);
- upsertStmt.setString(2, "DUMMY");
- upsertStmt.executeUpdate();
- conn.commit();
- conn.close();
-
- // Create a tenant-specific connection.
- props.setProperty("TenantId", "15");
- conn = DriverManager.getConnection(getUrl(), props);
- ResultSet rs = conn.createStatement().executeQuery(
- "SELECT * FROM INVENTORY INNER JOIN PRODUCT_IDS ON (PRODUCT_ID = INVENTORY.ID)");
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 5);
- assertFalse(rs.next());
- rs = conn.createStatement().executeQuery(
- "SELECT * FROM INVENTORY RIGHT JOIN PRODUCT_IDS ON (PRODUCT_ID = INVENTORY.ID)");
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 5);
- assertFalse(rs.next());
- } finally {
- conn.close();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/bc9974ba/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinMoreIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinMoreIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinMoreIT.java
new file mode 100644
index 0000000..a8bb977
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinMoreIT.java
@@ -0,0 +1,678 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Array;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+public class HashJoinMoreIT extends BaseHBaseManagedTimeIT {
+ private final String[] plans = new String[] {
+ /*
+ * testJoinWithKeyRangeOptimization()
+ * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
+ * FROM TEMP_TABLE_COMPOSITE_PK lhs
+ * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col1 = rhs.col2
+ */
+ "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
+ "CLIENT MERGE SORT\n" +
+ " PARALLEL INNER-JOIN TABLE 0\n" +
+ " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
+ " CLIENT MERGE SORT",
+ /*
+ * testJoinWithKeyRangeOptimization()
+ * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
+ * FROM TEMP_TABLE_COMPOSITE_PK lhs
+ * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col0 = rhs.col2
+ */
+ "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
+ "CLIENT MERGE SORT\n" +
+ " PARALLEL INNER-JOIN TABLE 0\n" +
+ " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
+ " CLIENT MERGE SORT\n" +
+ " DYNAMIC SERVER FILTER BY LHS.COL0 IN (RHS.COL2)",
+ /*
+ * testJoinWithKeyRangeOptimization()
+ * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
+ * FROM TEMP_TABLE_COMPOSITE_PK lhs
+ * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col0 = rhs.col1 AND lhs.col1 = rhs.col2
+ */
+ "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
+ "CLIENT MERGE SORT\n" +
+ " PARALLEL INNER-JOIN TABLE 0\n" +
+ " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
+ " CLIENT MERGE SORT\n" +
+ " DYNAMIC SERVER FILTER BY (LHS.COL0, LHS.COL1) IN ((RHS.COL1, RHS.COL2))",
+ /*
+ * testJoinWithKeyRangeOptimization()
+ * SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, rhs.col2
+ * FROM TEMP_TABLE_COMPOSITE_PK lhs
+ * JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col0 = rhs.col1 AND lhs.col2 = rhs.col3 - 1 AND lhs.col1 = rhs.col2
+ */
+ "CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
+ "CLIENT MERGE SORT\n" +
+ " PARALLEL INNER-JOIN TABLE 0\n" +
+ " CLIENT PARALLEL 4-WAY FULL SCAN OVER TEMP_TABLE_COMPOSITE_PK\n" +
+ " CLIENT MERGE SORT\n" +
+ " DYNAMIC SERVER FILTER BY (LHS.COL0, LHS.COL1, LHS.COL2) IN ((RHS.COL1, RHS.COL2, TO_INTEGER((RHS.COL3 - 1))))",
+ };
+
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
+ // Forces server cache to be used
+ props.put(QueryServices.INDEX_MUTATE_BATCH_SIZE_THRESHOLD_ATTRIB, Integer.toString(2));
+ // Must update config before starting server
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @Test
+ public void testJoinOverSaltedTables() throws Exception {
+ String tempTableNoSalting = "TEMP_TABLE_NO_SALTING";
+ String tempTableWithSalting = "TEMP_TABLE_WITH_SALTING";
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ try {
+ conn.createStatement().execute("CREATE TABLE " + tempTableNoSalting
+ + " (mypk INTEGER NOT NULL PRIMARY KEY, "
+ + " col1 INTEGER)");
+ conn.createStatement().execute("CREATE TABLE " + tempTableWithSalting
+ + " (mypk INTEGER NOT NULL PRIMARY KEY, "
+ + " col1 INTEGER) SALT_BUCKETS=4");
+
+ PreparedStatement upsertStmt = conn.prepareStatement(
+ "upsert into " + tempTableNoSalting + "(mypk, col1) " + "values (?, ?)");
+ for (int i = 0; i < 3; i++) {
+ upsertStmt.setInt(1, i + 1);
+ upsertStmt.setInt(2, 3 - i);
+ upsertStmt.execute();
+ }
+ conn.commit();
+
+ upsertStmt = conn.prepareStatement(
+ "upsert into " + tempTableWithSalting + "(mypk, col1) " + "values (?, ?)");
+ for (int i = 0; i < 6; i++) {
+ upsertStmt.setInt(1, i + 1);
+ upsertStmt.setInt(2, 3 - (i % 3));
+ upsertStmt.execute();
+ }
+ conn.commit();
+
+ // LHS=unsalted JOIN RHS=salted
+ String query = "SELECT lhs.mypk, lhs.col1, rhs.mypk, rhs.col1 FROM "
+ + tempTableNoSalting + " lhs JOIN "
+ + tempTableWithSalting + " rhs ON rhs.mypk = lhs.col1";
+ PreparedStatement statement = conn.prepareStatement(query);
+ ResultSet rs = statement.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 1);
+ assertEquals(rs.getInt(2), 3);
+ assertEquals(rs.getInt(3), 3);
+ assertEquals(rs.getInt(4), 1);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 2);
+ assertEquals(rs.getInt(2), 2);
+ assertEquals(rs.getInt(3), 2);
+ assertEquals(rs.getInt(4), 2);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 3);
+ assertEquals(rs.getInt(2), 1);
+ assertEquals(rs.getInt(3), 1);
+ assertEquals(rs.getInt(4), 3);
+
+ assertFalse(rs.next());
+
+ // LHS=salted JOIN RHS=salted
+ query = "SELECT lhs.mypk, lhs.col1, rhs.mypk, rhs.col1 FROM "
+ + tempTableWithSalting + " lhs JOIN "
+ + tempTableNoSalting + " rhs ON rhs.mypk = lhs.col1";
+ statement = conn.prepareStatement(query);
+ rs = statement.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 1);
+ assertEquals(rs.getInt(2), 3);
+ assertEquals(rs.getInt(3), 3);
+ assertEquals(rs.getInt(4), 1);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 2);
+ assertEquals(rs.getInt(2), 2);
+ assertEquals(rs.getInt(3), 2);
+ assertEquals(rs.getInt(4), 2);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 3);
+ assertEquals(rs.getInt(2), 1);
+ assertEquals(rs.getInt(3), 1);
+ assertEquals(rs.getInt(4), 3);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 4);
+ assertEquals(rs.getInt(2), 3);
+ assertEquals(rs.getInt(3), 3);
+ assertEquals(rs.getInt(4), 1);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 5);
+ assertEquals(rs.getInt(2), 2);
+ assertEquals(rs.getInt(3), 2);
+ assertEquals(rs.getInt(4), 2);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 6);
+ assertEquals(rs.getInt(2), 1);
+ assertEquals(rs.getInt(3), 1);
+ assertEquals(rs.getInt(4), 3);
+
+ assertFalse(rs.next());
+
+ // LHS=salted JOIN RHS=salted
+ query = "SELECT lhs.mypk, lhs.col1, rhs.mypk, rhs.col1 FROM "
+ + tempTableWithSalting + " lhs JOIN "
+ + tempTableWithSalting + " rhs ON rhs.mypk = (lhs.col1 + 3)";
+ statement = conn.prepareStatement(query);
+ rs = statement.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 1);
+ assertEquals(rs.getInt(2), 3);
+ assertEquals(rs.getInt(3), 6);
+ assertEquals(rs.getInt(4), 1);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 2);
+ assertEquals(rs.getInt(2), 2);
+ assertEquals(rs.getInt(3), 5);
+ assertEquals(rs.getInt(4), 2);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 3);
+ assertEquals(rs.getInt(2), 1);
+ assertEquals(rs.getInt(3), 4);
+ assertEquals(rs.getInt(4), 3);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 4);
+ assertEquals(rs.getInt(2), 3);
+ assertEquals(rs.getInt(3), 6);
+ assertEquals(rs.getInt(4), 1);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 5);
+ assertEquals(rs.getInt(2), 2);
+ assertEquals(rs.getInt(3), 5);
+ assertEquals(rs.getInt(4), 2);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 6);
+ assertEquals(rs.getInt(2), 1);
+ assertEquals(rs.getInt(3), 4);
+ assertEquals(rs.getInt(4), 3);
+
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testJoinOnDynamicColumns() throws Exception {
+ String tableA = "tableA";
+ String tableB = "tableB";
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = null;
+ PreparedStatement stmt = null;
+ try {
+ conn = DriverManager.getConnection(getUrl(), props);
+ String ddlA = "CREATE TABLE " + tableA + " (pkA INTEGER NOT NULL, " + " colA1 INTEGER, "
+ + " colA2 VARCHAR " + "CONSTRAINT PK PRIMARY KEY" + "(pkA)" + ")";
+
+ String ddlB = "CREATE TABLE " + tableB + " (pkB INTEGER NOT NULL PRIMARY KEY, " + " colB INTEGER)";
+ stmt = conn.prepareStatement(ddlA);
+ stmt.execute();
+ stmt.close();
+
+ stmt = conn.prepareStatement(ddlB);
+ stmt.execute();
+ stmt.close();
+
+ String upsertA = "UPSERT INTO TABLEA (pkA, colA1, colA2) VALUES(?, ?, ?)";
+ stmt = conn.prepareStatement(upsertA);
+ int i = 0;
+ for (i = 0; i < 5; i++) {
+ stmt.setInt(1, i);
+ stmt.setInt(2, i + 10);
+ stmt.setString(3, "00" + i);
+ stmt.executeUpdate();
+ }
+ conn.commit();
+ stmt.close();
+
+ // upsert select dynamic columns in tableB
+ conn.createStatement().execute("CREATE SEQUENCE SEQB");
+ String upsertBSelectA = "UPSERT INTO TABLEB (pkB, pkA INTEGER)"
+ + "SELECT NEXT VALUE FOR SEQB, pkA FROM TABLEA";
+ stmt = conn.prepareStatement(upsertBSelectA);
+ stmt.executeUpdate();
+ stmt.close();
+ conn.commit();
+ conn.createStatement().execute("DROP SEQUENCE SEQB");
+
+ // perform a join between tableB and tableA by joining on the dynamic column that we upserted in
+ // tableB. This join should return all the rows from table A.
+ String joinSql = "SELECT A.pkA, A.COLA1, A.colA2 FROM TABLEB B(pkA INTEGER) JOIN TABLEA A ON a.pkA = b.pkA";
+ stmt = conn.prepareStatement(joinSql);
+ ResultSet rs = stmt.executeQuery();
+ i = 0;
+ while (rs.next()) {
+ // check that we get back all the rows that we upserted for tableA above.
+ assertEquals(rs.getInt(1), i);
+ assertEquals(rs.getInt(2), i + 10);
+ assertEquals(rs.getString(3), "00" + i);
+ i++;
+ }
+ assertEquals(5,i);
+ } finally {
+ if (stmt != null) {
+ stmt.close();
+ }
+ if (conn != null) {
+ conn.close();
+ }
+
+ }
+
+ }
+
+ @Test
+ public void testJoinWithKeyRangeOptimization() throws Exception {
+ String tempTableWithCompositePK = "TEMP_TABLE_COMPOSITE_PK";
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ try {
+ conn.createStatement().execute("CREATE TABLE " + tempTableWithCompositePK
+ + " (col0 INTEGER NOT NULL, "
+ + " col1 INTEGER NOT NULL, "
+ + " col2 INTEGER NOT NULL, "
+ + " col3 INTEGER "
+ + " CONSTRAINT pk PRIMARY KEY (col0, col1, col2)) "
+ + " SALT_BUCKETS=4");
+
+ PreparedStatement upsertStmt = conn.prepareStatement(
+ "upsert into " + tempTableWithCompositePK + "(col0, col1, col2, col3) " + "values (?, ?, ?, ?)");
+ for (int i = 0; i < 3; i++) {
+ upsertStmt.setInt(1, i + 1);
+ upsertStmt.setInt(2, i + 2);
+ upsertStmt.setInt(3, i + 3);
+ upsertStmt.setInt(4, i + 5);
+ upsertStmt.execute();
+ }
+ conn.commit();
+
+ // No leading part of PK
+ String query = "SELECT lhs.col0, lhs.col1, lhs.col2, lhs.col3, rhs.col0, rhs.col1, rhs.col2, rhs.col3 FROM "
+ + tempTableWithCompositePK + " lhs JOIN "
+ + tempTableWithCompositePK + " rhs ON lhs.col1 = rhs.col2";
+ PreparedStatement statement = conn.prepareStatement(query);
+ ResultSet rs = statement.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 2);
+ assertEquals(rs.getInt(2), 3);
+ assertEquals(rs.getInt(3), 4);
+ assertEquals(rs.getInt(4), 6);
+ assertEquals(rs.getInt(5), 1);
+ assertEquals(rs.getInt(6), 2);
+ assertEquals(rs.getInt(7), 3);
+ assertEquals(rs.getInt(8), 5);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 3);
+ assertEquals(rs.getInt(2), 4);
+ assertEquals(rs.getInt(3), 5);
+ assertEquals(rs.getInt(4), 7);
+ assertEquals(rs.getInt(5), 2);
+ assertEquals(rs.getInt(6), 3);
+ assertEquals(rs.getInt(7), 4);
+ assertEquals(rs.getInt(8), 6);
+
+ assertFalse(rs.next());
+
+ rs = conn.createStatement().executeQuery("EXPLAIN " + query);
+ assertEquals(plans[0], QueryUtil.getExplainPlan(rs));
+
+ // Two parts of PK but only one leading part
+ query = "SELECT lhs.col0, lhs.col1, lhs.col2, lhs.col3, rhs.col0, rhs.col1, rhs.col2, rhs.col3 FROM "
+ + tempTableWithCompositePK + " lhs JOIN "
+ + tempTableWithCompositePK + " rhs ON lhs.col2 = rhs.col3 AND lhs.col0 = rhs.col2";
+ statement = conn.prepareStatement(query);
+ rs = statement.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 3);
+ assertEquals(rs.getInt(2), 4);
+ assertEquals(rs.getInt(3), 5);
+ assertEquals(rs.getInt(4), 7);
+ assertEquals(rs.getInt(5), 1);
+ assertEquals(rs.getInt(6), 2);
+ assertEquals(rs.getInt(7), 3);
+ assertEquals(rs.getInt(8), 5);
+
+ assertFalse(rs.next());
+
+ rs = conn.createStatement().executeQuery("EXPLAIN " + query);
+ assertEquals(plans[1], QueryUtil.getExplainPlan(rs));
+
+ // Two leading parts of PK
+ query = "SELECT lhs.col0, lhs.col1, lhs.col2, lhs.col3, rhs.col0, rhs.col1, rhs.col2, rhs.col3 FROM "
+ + tempTableWithCompositePK + " lhs JOIN "
+ + tempTableWithCompositePK + " rhs ON lhs.col1 = rhs.col2 AND lhs.col0 = rhs.col1";
+ statement = conn.prepareStatement(query);
+ rs = statement.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 2);
+ assertEquals(rs.getInt(2), 3);
+ assertEquals(rs.getInt(3), 4);
+ assertEquals(rs.getInt(4), 6);
+ assertEquals(rs.getInt(5), 1);
+ assertEquals(rs.getInt(6), 2);
+ assertEquals(rs.getInt(7), 3);
+ assertEquals(rs.getInt(8), 5);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 3);
+ assertEquals(rs.getInt(2), 4);
+ assertEquals(rs.getInt(3), 5);
+ assertEquals(rs.getInt(4), 7);
+ assertEquals(rs.getInt(5), 2);
+ assertEquals(rs.getInt(6), 3);
+ assertEquals(rs.getInt(7), 4);
+ assertEquals(rs.getInt(8), 6);
+
+ assertFalse(rs.next());
+
+ rs = conn.createStatement().executeQuery("EXPLAIN " + query);
+ assertEquals(plans[2], QueryUtil.getExplainPlan(rs));
+
+ // All parts of PK
+ query = "SELECT lhs.col0, lhs.col1, lhs.col2, lhs.col3, rhs.col0, rhs.col1, rhs.col2, rhs.col3 FROM "
+ + tempTableWithCompositePK + " lhs JOIN "
+ + tempTableWithCompositePK + " rhs ON lhs.col1 = rhs.col2 AND lhs.col2 = rhs.col3 - 1 AND lhs.col0 = rhs.col1";
+ statement = conn.prepareStatement(query);
+ rs = statement.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 2);
+ assertEquals(rs.getInt(2), 3);
+ assertEquals(rs.getInt(3), 4);
+ assertEquals(rs.getInt(4), 6);
+ assertEquals(rs.getInt(5), 1);
+ assertEquals(rs.getInt(6), 2);
+ assertEquals(rs.getInt(7), 3);
+ assertEquals(rs.getInt(8), 5);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 3);
+ assertEquals(rs.getInt(2), 4);
+ assertEquals(rs.getInt(3), 5);
+ assertEquals(rs.getInt(4), 7);
+ assertEquals(rs.getInt(5), 2);
+ assertEquals(rs.getInt(6), 3);
+ assertEquals(rs.getInt(7), 4);
+ assertEquals(rs.getInt(8), 6);
+
+ assertFalse(rs.next());
+
+ rs = conn.createStatement().executeQuery("EXPLAIN " + query);
+ assertEquals(plans[3], QueryUtil.getExplainPlan(rs));
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testSubqueryWithoutData() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+
+ try {
+ String GRAMMAR_TABLE = "CREATE TABLE IF NOT EXISTS GRAMMAR_TABLE (ID INTEGER PRIMARY KEY, " +
+ "unsig_id UNSIGNED_INT, big_id BIGINT, unsig_long_id UNSIGNED_LONG, tiny_id TINYINT," +
+ "unsig_tiny_id UNSIGNED_TINYINT, small_id SMALLINT, unsig_small_id UNSIGNED_SMALLINT," +
+ "float_id FLOAT, unsig_float_id UNSIGNED_FLOAT, double_id DOUBLE, unsig_double_id UNSIGNED_DOUBLE," +
+ "decimal_id DECIMAL, boolean_id BOOLEAN, time_id TIME, date_id DATE, timestamp_id TIMESTAMP," +
+ "unsig_time_id TIME, unsig_date_id DATE, unsig_timestamp_id TIMESTAMP, varchar_id VARCHAR (30)," +
+ "char_id CHAR (30), binary_id BINARY (100), varbinary_id VARBINARY (100))";
+
+ String LARGE_TABLE = "CREATE TABLE IF NOT EXISTS LARGE_TABLE (ID INTEGER PRIMARY KEY, " +
+ "unsig_id UNSIGNED_INT, big_id BIGINT, unsig_long_id UNSIGNED_LONG, tiny_id TINYINT," +
+ "unsig_tiny_id UNSIGNED_TINYINT, small_id SMALLINT, unsig_small_id UNSIGNED_SMALLINT," +
+ "float_id FLOAT, unsig_float_id UNSIGNED_FLOAT, double_id DOUBLE, unsig_double_id UNSIGNED_DOUBLE," +
+ "decimal_id DECIMAL, boolean_id BOOLEAN, time_id TIME, date_id DATE, timestamp_id TIMESTAMP," +
+ "unsig_time_id TIME, unsig_date_id DATE, unsig_timestamp_id TIMESTAMP, varchar_id VARCHAR (30)," +
+ "char_id CHAR (30), binary_id BINARY (100), varbinary_id VARBINARY (100))";
+
+ String SECONDARY_LARGE_TABLE = "CREATE TABLE IF NOT EXISTS SECONDARY_LARGE_TABLE (SEC_ID INTEGER PRIMARY KEY," +
+ "sec_unsig_id UNSIGNED_INT, sec_big_id BIGINT, sec_usnig_long_id UNSIGNED_LONG, sec_tiny_id TINYINT," +
+ "sec_unsig_tiny_id UNSIGNED_TINYINT, sec_small_id SMALLINT, sec_unsig_small_id UNSIGNED_SMALLINT," +
+ "sec_float_id FLOAT, sec_unsig_float_id UNSIGNED_FLOAT, sec_double_id DOUBLE, sec_unsig_double_id UNSIGNED_DOUBLE," +
+ "sec_decimal_id DECIMAL, sec_boolean_id BOOLEAN, sec_time_id TIME, sec_date_id DATE," +
+ "sec_timestamp_id TIMESTAMP, sec_unsig_time_id TIME, sec_unsig_date_id DATE, sec_unsig_timestamp_id TIMESTAMP," +
+ "sec_varchar_id VARCHAR (30), sec_char_id CHAR (30), sec_binary_id BINARY (100), sec_varbinary_id VARBINARY (100))";
+ createTestTable(getUrl(), GRAMMAR_TABLE);
+ createTestTable(getUrl(), LARGE_TABLE);
+ createTestTable(getUrl(), SECONDARY_LARGE_TABLE);
+
+ String ddl = "SELECT * FROM (SELECT ID, BIG_ID, DATE_ID FROM LARGE_TABLE AS A WHERE (A.ID % 5) = 0) AS A " +
+ "INNER JOIN (SELECT SEC_ID, SEC_TINY_ID, SEC_UNSIG_FLOAT_ID FROM SECONDARY_LARGE_TABLE AS B WHERE (B.SEC_ID % 5) = 0) AS B " +
+ "ON A.ID=B.SEC_ID WHERE A.DATE_ID > ALL (SELECT SEC_DATE_ID FROM SECONDARY_LARGE_TABLE LIMIT 100) " +
+ "AND B.SEC_UNSIG_FLOAT_ID = ANY (SELECT sec_unsig_float_id FROM SECONDARY_LARGE_TABLE " +
+ "WHERE SEC_ID > ALL (SELECT MIN (ID) FROM GRAMMAR_TABLE WHERE UNSIG_ID IS NULL) AND " +
+ "SEC_UNSIG_ID < ANY (SELECT DISTINCT(UNSIG_ID) FROM LARGE_TABLE WHERE UNSIG_ID<2500) LIMIT 1000) " +
+ "AND A.ID < 10000";
+ ResultSet rs = conn.createStatement().executeQuery(ddl);
+ assertFalse(rs.next());
+ } finally {
+ Statement statement = conn.createStatement();
+ String query = "drop table GRAMMAR_TABLE";
+ statement.executeUpdate(query);
+ query = "drop table LARGE_TABLE";
+ statement.executeUpdate(query);
+ query = "drop table SECONDARY_LARGE_TABLE";
+ statement.executeUpdate(query);
+ conn.close();
+ }
+ }
+
+ // PHOENIX-2381
+ @Test
+ public void testJoinWithMultiTenancy() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ try {
+ conn.createStatement().execute("CREATE TABLE INVENTORY (" +
+ " TENANTID UNSIGNED_INT NOT NULL" +
+ ",ID UNSIGNED_INT NOT NULL" +
+ ",FOO UNSIGNED_INT NOT NULL" +
+ ",TIMESTAMP UNSIGNED_LONG NOT NULL" +
+ ",CODES INTEGER ARRAY[] NOT NULL" +
+ ",V UNSIGNED_LONG" +
+ " CONSTRAINT pk PRIMARY KEY (TENANTID, ID, FOO, TIMESTAMP, CODES))" +
+ " DEFAULT_COLUMN_FAMILY ='E'," +
+ " MULTI_TENANT=true");
+ PreparedStatement upsertStmt = conn.prepareStatement(
+ "upsert into INVENTORY "
+ + "(tenantid, id, foo, timestamp, codes) "
+ + "values (?, ?, ?, ?, ?)");
+ upsertStmt.setInt(1, 15);
+ upsertStmt.setInt(2, 5);
+ upsertStmt.setInt(3, 0);
+ upsertStmt.setLong(4, 6);
+ Array array = conn.createArrayOf("INTEGER", new Object[] {1, 2});
+ upsertStmt.setArray(5, array);
+ upsertStmt.executeUpdate();
+ conn.commit();
+
+ conn.createStatement().execute("CREATE TABLE PRODUCT_IDS (" +
+ " PRODUCT_ID UNSIGNED_INT NOT NULL" +
+ ",PRODUCT_NAME VARCHAR" +
+ " CONSTRAINT pk PRIMARY KEY (PRODUCT_ID))" +
+ " DEFAULT_COLUMN_FAMILY ='NAME'");
+ upsertStmt = conn.prepareStatement(
+ "upsert into PRODUCT_IDS "
+ + "(product_id, product_name) "
+ + "values (?, ?)");
+ upsertStmt.setInt(1, 5);
+ upsertStmt.setString(2, "DUMMY");
+ upsertStmt.executeUpdate();
+ conn.commit();
+ conn.close();
+
+ // Create a tenant-specific connection.
+ props.setProperty("TenantId", "15");
+ conn = DriverManager.getConnection(getUrl(), props);
+ ResultSet rs = conn.createStatement().executeQuery(
+ "SELECT * FROM INVENTORY INNER JOIN PRODUCT_IDS ON (PRODUCT_ID = INVENTORY.ID)");
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 5);
+ assertFalse(rs.next());
+ rs = conn.createStatement().executeQuery(
+ "SELECT * FROM INVENTORY RIGHT JOIN PRODUCT_IDS ON (PRODUCT_ID = INVENTORY.ID)");
+ assertTrue(rs.next());
+ assertEquals(rs.getInt(1), 5);
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testBug2480() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(true);
+ try {
+ conn.createStatement().execute(
+ "CREATE TABLE master_businessunit ("
+ + "code varchar(255) PRIMARY KEY, name varchar(255))");
+ conn.createStatement().execute(
+ "CREATE TABLE master_company ("
+ + "code varchar(255) PRIMARY KEY, name varchar(255))");
+ conn.createStatement().execute(
+ "CREATE TABLE master_costcenter ("
+ + "code varchar(255) PRIMARY KEY, name varchar(255))");
+ conn.createStatement().execute(
+ "CREATE TABLE master_location ("
+ + "code varchar(255) PRIMARY KEY, name varchar(255))");
+ conn.createStatement().execute(
+ "CREATE TABLE master_product ("
+ + "id integer PRIMARY KEY, product_name varchar(255))");
+ conn.createStatement().execute(
+ "CREATE TABLE master_purchaseorder ("
+ + "purchaseOrderNumber varchar(255), "
+ + "companyCode varchar(255), "
+ + "businessUnitCode varchar(255), "
+ + "locationCode varchar(255), "
+ + "purchaseOrderId varchar(255) PRIMARY KEY, "
+ + "releasedOn date, name varchar(255))");
+ conn.createStatement().execute(
+ "CREATE TABLE trans_purchaseorderitem ("
+ + "purchaseOrderItemId varchar(255) PRIMARY KEY, "
+ + "purchaseOrderId varchar(255), "
+ + "lineNo varchar(255), name varchar(255))");
+ conn.createStatement().execute(
+ "CREATE TABLE trans_purchaseorderitem_costing ("
+ + "purchaseorderItem_costing_id varchar(255) primary key, "
+ + "purchaseorderItemId varchar(255), "
+ + "purchaseorderId varchar(255), "
+ + "costcenterCode varchar(255))");
+
+ conn.createStatement().execute("upsert into master_businessunit(code,name) values ('1','BU1')");
+ conn.createStatement().execute("upsert into master_businessunit(code,name) values ('2','BU2')");
+ conn.createStatement().execute("upsert into master_company(code,name) values ('1','Company1')");
+ conn.createStatement().execute("upsert into master_company(code,name) values ('2','Company2')");
+ conn.createStatement().execute("upsert into master_costcenter(code,name) values ('1','CC1')");
+ conn.createStatement().execute("upsert into master_costcenter(code,name) values ('2','CC2')");
+ conn.createStatement().execute("upsert into master_location(code,name) values ('1','Location1')");
+ conn.createStatement().execute("upsert into master_location(code,name) values ('2','Location2')");
+ conn.createStatement().execute("upsert into master_product(id,product_name) values (1,'ProductName1')");
+ conn.createStatement().execute("upsert into master_product(id,product_name) values (2,'Product2')");
+ conn.createStatement().execute("upsert into master_purchaseorder(purchaseOrderNumber,companyCode,businessUnitCode,locationCode,purchaseOrderId,releasedOn,name) values ('1','1','1','1','1','2015-12-01','1')");
+ conn.createStatement().execute("upsert into master_purchaseorder(purchaseOrderNumber,companyCode,businessUnitCode,locationCode,purchaseOrderId,releasedOn,name) values ('2','2','2','2','2','2015-12-02','2')");
+ conn.createStatement().execute("upsert into trans_purchaseorderitem(purchaseOrderItemId,purchaseOrderId,lineNo,name) values ('1','1','1','1')");
+ conn.createStatement().execute("upsert into trans_purchaseorderitem(purchaseOrderItemId,purchaseOrderId,lineNo,name) values ('2','2','2','2')");
+ conn.createStatement().execute("upsert into trans_purchaseorderitem_costing(purchaseorderItem_costing_id,purchaseorderItemId,purchaseorderId,costcenterCode) values ('1','1','1','1')");
+ conn.createStatement().execute("upsert into trans_purchaseorderitem_costing(purchaseorderItem_costing_id,purchaseorderItemId,purchaseorderId,costcenterCode) values ('2','2','2','2')");
+
+ ResultSet rs = conn.createStatement().executeQuery(
+ "SELECT DISTINCT "
+ + "COALESCE( a1.name, 'N.A.'), "
+ + "COALESCE( a2.name, 'N.A.'), "
+ + "COALESCE( a3.name, 'N.A.'), "
+ + "COALESCE( a4.purchaseOrderNumber, 'N.A.'), "
+ + "COALESCE( a1.name, 'N.A.'), "
+ + "COALESCE( a4.name, 'N.A.'), "
+ + "COALESCE( a5.lineNo, 'N.A.'), "
+ + "COALESCE( a5.name, 'N.A.'), "
+ + "COALESCE( a7.name,'N.A.') "
+ + "FROM (master_purchaseorder a4 "
+ + "LEFT OUTER JOIN master_company a1 "
+ + "ON a4.companyCode = a1.code "
+ + "LEFT OUTER JOIN master_businessunit a2 "
+ + "ON a4.businessUnitCode = a2.code "
+ + "LEFT OUTER JOIN master_location a3 "
+ + "ON a4.locationCode = a3.code "
+ + "LEFT OUTER JOIN trans_purchaseorderitem a5 "
+ + "ON a5.purchaseOrderId = a4.purchaseOrderId "
+ + "LEFT OUTER JOIN trans_purchaseorderitem_costing a6 "
+ + "ON a6.purchaseOrderItemId = a5.purchaseOrderItemId "
+ + "AND a6.purchaseOrderId = a5.purchaseOrderId "
+ + "LEFT OUTER JOIN master_costcenter a7 "
+ + "ON a6.costCenterCode = a7.code)");
+
+ assertTrue(rs.next());
+ assertEquals("Company1", rs.getString(1));
+ assertEquals("BU1", rs.getString(2));
+ assertEquals("Location1", rs.getString(3));
+ assertEquals("1", rs.getString(4));
+ assertEquals("Company1", rs.getString(5));
+ assertEquals("1", rs.getString(6));
+ assertEquals("1", rs.getString(7));
+ assertEquals("1", rs.getString(8));
+ assertEquals("CC1", rs.getString(9));
+ assertTrue(rs.next());
+ assertEquals("Company2", rs.getString(1));
+ assertEquals("BU2", rs.getString(2));
+ assertEquals("Location2", rs.getString(3));
+ assertEquals("2", rs.getString(4));
+ assertEquals("Company2", rs.getString(5));
+ assertEquals("2", rs.getString(6));
+ assertEquals("2", rs.getString(7));
+ assertEquals("2", rs.getString(8));
+ assertEquals("CC2", rs.getString(9));
+
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/bc9974ba/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinIT.java
index 9b85d8a..f19b886 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinIT.java
@@ -1887,213 +1887,6 @@ public class SortMergeJoinIT extends BaseHBaseManagedTimeIT {
}
@Test
- public void testJoinOverSaltedTables() throws Exception {
- String tempTableNoSalting = "TEMP_TABLE_NO_SALTING";
- String tempTableWithSalting = "TEMP_TABLE_WITH_SALTING";
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- try {
- conn.createStatement().execute("CREATE TABLE " + tempTableNoSalting
- + " (mypk INTEGER NOT NULL PRIMARY KEY, "
- + " col1 INTEGER)");
- conn.createStatement().execute("CREATE TABLE " + tempTableWithSalting
- + " (mypk INTEGER NOT NULL PRIMARY KEY, "
- + " col1 INTEGER) SALT_BUCKETS=4");
-
- PreparedStatement upsertStmt = conn.prepareStatement(
- "upsert into " + tempTableNoSalting + "(mypk, col1) " + "values (?, ?)");
- for (int i = 0; i < 3; i++) {
- upsertStmt.setInt(1, i + 1);
- upsertStmt.setInt(2, 3 - i);
- upsertStmt.execute();
- }
- conn.commit();
-
- upsertStmt = conn.prepareStatement(
- "upsert into " + tempTableWithSalting + "(mypk, col1) " + "values (?, ?)");
- for (int i = 0; i < 6; i++) {
- upsertStmt.setInt(1, i + 1);
- upsertStmt.setInt(2, 3 - (i % 3));
- upsertStmt.execute();
- }
- conn.commit();
-
- // LHS=unsalted JOIN RHS=salted
- String query = "SELECT /*+ USE_SORT_MERGE_JOIN*/ lhs.mypk, lhs.col1, rhs.mypk, rhs.col1 FROM "
- + tempTableNoSalting + " lhs JOIN "
- + tempTableWithSalting + " rhs ON rhs.mypk = lhs.col1 ORDER BY lhs.mypk";
- PreparedStatement statement = conn.prepareStatement(query);
- ResultSet rs = statement.executeQuery();
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 1);
- assertEquals(rs.getInt(2), 3);
- assertEquals(rs.getInt(3), 3);
- assertEquals(rs.getInt(4), 1);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 2);
- assertEquals(rs.getInt(2), 2);
- assertEquals(rs.getInt(3), 2);
- assertEquals(rs.getInt(4), 2);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 3);
- assertEquals(rs.getInt(2), 1);
- assertEquals(rs.getInt(3), 1);
- assertEquals(rs.getInt(4), 3);
-
- assertFalse(rs.next());
-
- // LHS=salted JOIN RHS=salted
- query = "SELECT /*+ USE_SORT_MERGE_JOIN*/ lhs.mypk, lhs.col1, rhs.mypk, rhs.col1 FROM "
- + tempTableWithSalting + " lhs JOIN "
- + tempTableNoSalting + " rhs ON rhs.mypk = lhs.col1 ORDER BY lhs.mypk";
- statement = conn.prepareStatement(query);
- rs = statement.executeQuery();
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 1);
- assertEquals(rs.getInt(2), 3);
- assertEquals(rs.getInt(3), 3);
- assertEquals(rs.getInt(4), 1);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 2);
- assertEquals(rs.getInt(2), 2);
- assertEquals(rs.getInt(3), 2);
- assertEquals(rs.getInt(4), 2);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 3);
- assertEquals(rs.getInt(2), 1);
- assertEquals(rs.getInt(3), 1);
- assertEquals(rs.getInt(4), 3);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 4);
- assertEquals(rs.getInt(2), 3);
- assertEquals(rs.getInt(3), 3);
- assertEquals(rs.getInt(4), 1);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 5);
- assertEquals(rs.getInt(2), 2);
- assertEquals(rs.getInt(3), 2);
- assertEquals(rs.getInt(4), 2);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 6);
- assertEquals(rs.getInt(2), 1);
- assertEquals(rs.getInt(3), 1);
- assertEquals(rs.getInt(4), 3);
-
- assertFalse(rs.next());
-
- // LHS=salted JOIN RHS=salted
- query = "SELECT /*+ USE_SORT_MERGE_JOIN*/ lhs.mypk, lhs.col1, rhs.mypk, rhs.col1 FROM "
- + tempTableWithSalting + " lhs JOIN "
- + tempTableWithSalting + " rhs ON rhs.mypk = (lhs.col1 + 3) ORDER BY lhs.mypk";
- statement = conn.prepareStatement(query);
- rs = statement.executeQuery();
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 1);
- assertEquals(rs.getInt(2), 3);
- assertEquals(rs.getInt(3), 6);
- assertEquals(rs.getInt(4), 1);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 2);
- assertEquals(rs.getInt(2), 2);
- assertEquals(rs.getInt(3), 5);
- assertEquals(rs.getInt(4), 2);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 3);
- assertEquals(rs.getInt(2), 1);
- assertEquals(rs.getInt(3), 4);
- assertEquals(rs.getInt(4), 3);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 4);
- assertEquals(rs.getInt(2), 3);
- assertEquals(rs.getInt(3), 6);
- assertEquals(rs.getInt(4), 1);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 5);
- assertEquals(rs.getInt(2), 2);
- assertEquals(rs.getInt(3), 5);
- assertEquals(rs.getInt(4), 2);
- assertTrue(rs.next());
- assertEquals(rs.getInt(1), 6);
- assertEquals(rs.getInt(2), 1);
- assertEquals(rs.getInt(3), 4);
- assertEquals(rs.getInt(4), 3);
-
- assertFalse(rs.next());
- } finally {
- conn.close();
- }
- }
-
- @Test
- public void testJoinOnDynamicColumns() throws Exception {
- String tableA = "tableA";
- String tableB = "tableB";
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = null;
- PreparedStatement stmt = null;
- try {
- conn = DriverManager.getConnection(getUrl(), props);
- String ddlA = "CREATE TABLE " + tableA + " (pkA INTEGER NOT NULL, " + " colA1 INTEGER, "
- + " colA2 VARCHAR " + "CONSTRAINT PK PRIMARY KEY" + "(pkA)" + ")";
-
- String ddlB = "CREATE TABLE " + tableB + " (pkB INTEGER NOT NULL PRIMARY KEY, " + " colB INTEGER)";
- stmt = conn.prepareStatement(ddlA);
- stmt.execute();
- stmt.close();
-
- stmt = conn.prepareStatement(ddlB);
- stmt.execute();
- stmt.close();
-
- String upsertA = "UPSERT INTO TABLEA (pkA, colA1, colA2) VALUES(?, ?, ?)";
- stmt = conn.prepareStatement(upsertA);
- int i = 0;
- for (i = 0; i < 5; i++) {
- stmt.setInt(1, i);
- stmt.setInt(2, i + 10);
- stmt.setString(3, "00" + i);
- stmt.executeUpdate();
- }
- conn.commit();
- stmt.close();
-
- // upsert select dynamic columns in tableB
- conn.createStatement().execute("CREATE SEQUENCE SEQB");
- String upsertBSelectA = "UPSERT INTO TABLEB (pkB, pkA INTEGER)"
- + "SELECT NEXT VALUE FOR SEQB, pkA FROM TABLEA";
- stmt = conn.prepareStatement(upsertBSelectA);
- stmt.executeUpdate();
- stmt.close();
- conn.commit();
- conn.createStatement().execute("DROP SEQUENCE SEQB");
-
- // perform a join between tableB and tableA by joining on the dynamic column that we upserted in
- // tableB. This join should return all the rows from table A.
- String joinSql = "SELECT /*+ USE_SORT_MERGE_JOIN*/ A.pkA, A.COLA1, A.colA2 FROM TABLEB B(pkA INTEGER) JOIN TABLEA A ON a.pkA = b.pkA";
- stmt = conn.prepareStatement(joinSql);
- ResultSet rs = stmt.executeQuery();
- i = 0;
- while (rs.next()) {
- // check that we get back all the rows that we upserted for tableA above.
- assertEquals(rs.getInt(1), i);
- assertEquals(rs.getInt(2), i + 10);
- assertEquals(rs.getString(3), "00" + i);
- i++;
- }
- assertEquals(5,i);
- } finally {
- if (stmt != null) {
- stmt.close();
- }
- if (conn != null) {
- conn.close();
- }
-
- }
-
- }
-
- @Test
public void testSubJoin() throws Exception {
String query1 = "SELECT /*+ USE_SORT_MERGE_JOIN*/ i.name, count(c.name), min(s.name), max(quantity) FROM " + JOIN_ORDER_TABLE_FULL_NAME + " o LEFT JOIN "
+ "(" + JOIN_SUPPLIER_TABLE_FULL_NAME + " s RIGHT JOIN " + JOIN_ITEM_TABLE_FULL_NAME + " i ON i.\"supplier_id\" = s.\"supplier_id\")"
@@ -2688,60 +2481,5 @@ public class SortMergeJoinIT extends BaseHBaseManagedTimeIT {
conn.close();
}
}
-
- @Test
- public void testSubqueryWithoutData() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- conn.setAutoCommit(false);
-
- try {
- String GRAMMAR_TABLE = "CREATE TABLE IF NOT EXISTS GRAMMAR_TABLE (ID INTEGER PRIMARY KEY, " +
- "unsig_id UNSIGNED_INT, big_id BIGINT, unsig_long_id UNSIGNED_LONG, tiny_id TINYINT," +
- "unsig_tiny_id UNSIGNED_TINYINT, small_id SMALLINT, unsig_small_id UNSIGNED_SMALLINT," +
- "float_id FLOAT, unsig_float_id UNSIGNED_FLOAT, double_id DOUBLE, unsig_double_id UNSIGNED_DOUBLE," +
- "decimal_id DECIMAL, boolean_id BOOLEAN, time_id TIME, date_id DATE, timestamp_id TIMESTAMP," +
- "unsig_time_id TIME, unsig_date_id DATE, unsig_timestamp_id TIMESTAMP, varchar_id VARCHAR (30)," +
- "char_id CHAR (30), binary_id BINARY (100), varbinary_id VARBINARY (100))";
-
- String LARGE_TABLE = "CREATE TABLE IF NOT EXISTS LARGE_TABLE (ID INTEGER PRIMARY KEY, " +
- "unsig_id UNSIGNED_INT, big_id BIGINT, unsig_long_id UNSIGNED_LONG, tiny_id TINYINT," +
- "unsig_tiny_id UNSIGNED_TINYINT, small_id SMALLINT, unsig_small_id UNSIGNED_SMALLINT," +
- "float_id FLOAT, unsig_float_id UNSIGNED_FLOAT, double_id DOUBLE, unsig_double_id UNSIGNED_DOUBLE," +
- "decimal_id DECIMAL, boolean_id BOOLEAN, time_id TIME, date_id DATE, timestamp_id TIMESTAMP," +
- "unsig_time_id TIME, unsig_date_id DATE, unsig_timestamp_id TIMESTAMP, varchar_id VARCHAR (30)," +
- "char_id CHAR (30), binary_id BINARY (100), varbinary_id VARBINARY (100))";
-
- String SECONDARY_LARGE_TABLE = "CREATE TABLE IF NOT EXISTS SECONDARY_LARGE_TABLE (SEC_ID INTEGER PRIMARY KEY," +
- "sec_unsig_id UNSIGNED_INT, sec_big_id BIGINT, sec_usnig_long_id UNSIGNED_LONG, sec_tiny_id TINYINT," +
- "sec_unsig_tiny_id UNSIGNED_TINYINT, sec_small_id SMALLINT, sec_unsig_small_id UNSIGNED_SMALLINT," +
- "sec_float_id FLOAT, sec_unsig_float_id UNSIGNED_FLOAT, sec_double_id DOUBLE, sec_unsig_double_id UNSIGNED_DOUBLE," +
- "sec_decimal_id DECIMAL, sec_boolean_id BOOLEAN, sec_time_id TIME, sec_date_id DATE," +
- "sec_timestamp_id TIMESTAMP, sec_unsig_time_id TIME, sec_unsig_date_id DATE, sec_unsig_timestamp_id TIMESTAMP," +
- "sec_varchar_id VARCHAR (30), sec_char_id CHAR (30), sec_binary_id BINARY (100), sec_varbinary_id VARBINARY (100))";
- createTestTable(getUrl(), GRAMMAR_TABLE);
- createTestTable(getUrl(), LARGE_TABLE);
- createTestTable(getUrl(), SECONDARY_LARGE_TABLE);
-
- String ddl = "SELECT /*+USE_SORT_MERGE_JOIN*/ * FROM (SELECT ID, BIG_ID, DATE_ID FROM LARGE_TABLE AS A WHERE (A.ID % 5) = 0) AS A " +
- "INNER JOIN (SELECT SEC_ID, SEC_TINY_ID, SEC_UNSIG_FLOAT_ID FROM SECONDARY_LARGE_TABLE AS B WHERE (B.SEC_ID % 5) = 0) AS B " +
- "ON A.ID=B.SEC_ID WHERE A.DATE_ID > ALL (SELECT SEC_DATE_ID FROM SECONDARY_LARGE_TABLE LIMIT 100) " +
- "AND B.SEC_UNSIG_FLOAT_ID = ANY (SELECT sec_unsig_float_id FROM SECONDARY_LARGE_TABLE " +
- "WHERE SEC_ID > ALL (SELECT MIN (ID) FROM GRAMMAR_TABLE WHERE UNSIG_ID IS NULL) AND " +
- "SEC_UNSIG_ID < ANY (SELECT DISTINCT(UNSIG_ID) FROM LARGE_TABLE WHERE UNSIG_ID<2500) LIMIT 1000) " +
- "AND A.ID < 10000";
- ResultSet rs = conn.createStatement().executeQuery(ddl);
- assertFalse(rs.next());
- } finally {
- Statement statement = conn.createStatement();
- String query = "drop table GRAMMAR_TABLE";
- statement.executeUpdate(query);
- query = "drop table LARGE_TABLE";
- statement.executeUpdate(query);
- query = "drop table SECONDARY_LARGE_TABLE";
- statement.executeUpdate(query);
- conn.close();
- }
- }
}
[41/52] [abbrv] phoenix git commit: PHOENIX-2448 Fix quoting for
sqlline-thin.py on Windows.
Posted by ma...@apache.org.
PHOENIX-2448 Fix quoting for sqlline-thin.py on Windows.
Also fix the use of "hbase" instead of "hbase.cmd" on Windows.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a8238f9b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a8238f9b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a8238f9b
Branch: refs/heads/calcite
Commit: a8238f9bcfd0c091d521f72fa224ba147b0acad3
Parents: 9478d1f
Author: Josh Elser <el...@apache.org>
Authored: Mon Nov 23 19:04:56 2015 -0500
Committer: Nick Dimiduk <nd...@apache.org>
Committed: Fri Dec 4 16:47:20 2015 -0800
----------------------------------------------------------------------
bin/sqlline-thin.py | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a8238f9b/bin/sqlline-thin.py
----------------------------------------------------------------------
diff --git a/bin/sqlline-thin.py b/bin/sqlline-thin.py
index 2e237ed..d17d756 100755
--- a/bin/sqlline-thin.py
+++ b/bin/sqlline-thin.py
@@ -57,7 +57,15 @@ def cleanup_url(url):
def get_serialization():
default_serialization='PROTOBUF'
env=os.environ.copy()
- hbase_cmd = phoenix_utils.which('hbase')
+ if os.name == 'posix':
+ hbase_exec_name = 'hbase'
+ elif os.name == 'nt':
+ hbase_exec_name = 'hbase.cmd'
+ else:
+ print 'Unknown platform "%s", defaulting to HBase executable of "hbase"' % os.name
+ hbase_exec_name = 'hbase'
+
+ hbase_cmd = phoenix_utils.which(hbase_exec_name)
if hbase_cmd is None:
print 'Failed to find hbase executable on PATH, defaulting serialization to %s.' % default_serialization
return default_serialization
@@ -141,7 +149,7 @@ java_cmd = java + ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix
os.pathsep + phoenix_utils.hadoop_conf + os.pathsep + phoenix_utils.hadoop_classpath + '" -Dlog4j.configuration=file:' + \
os.path.join(phoenix_utils.current_dir, "log4j.properties") + \
" sqlline.SqlLine -d org.apache.phoenix.queryserver.client.Driver " + \
- " -u jdbc:phoenix:thin:url='" + url + ";serialization=" + serialization + "'" + \
+ " -u \"jdbc:phoenix:thin:url=" + url + ";serialization=" + serialization + "\"" + \
" -n none -p none --color=" + colorSetting + " --fastConnect=false --verbose=true " + \
" --isolation=TRANSACTION_READ_COMMITTED " + sqlfile
[32/52] [abbrv] phoenix git commit: PHOENIX-2469 Problem with arrays
in phoenix-spark (Dawid Wysakowicz)
Posted by ma...@apache.org.
PHOENIX-2469 Problem with arrays in phoenix-spark (Dawid Wysakowicz)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b8faae52
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b8faae52
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b8faae52
Branch: refs/heads/calcite
Commit: b8faae52c6bee91393678e74de09ab8a215da856
Parents: 00ee941
Author: Josh Mahonin <jm...@interset.com>
Authored: Wed Dec 2 09:17:11 2015 -0500
Committer: Josh Mahonin <jm...@interset.com>
Committed: Wed Dec 2 09:17:11 2015 -0500
----------------------------------------------------------------------
phoenix-spark/pom.xml | 2 +-
phoenix-spark/src/it/resources/setup.sql | 7 ++
.../apache/phoenix/spark/PhoenixSparkIT.scala | 112 ++++++++++++++++++-
.../phoenix/spark/PhoenixRecordWritable.scala | 45 +++++---
4 files changed, 145 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b8faae52/phoenix-spark/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-spark/pom.xml b/phoenix-spark/pom.xml
index c0ad626..437f1c3 100644
--- a/phoenix-spark/pom.xml
+++ b/phoenix-spark/pom.xml
@@ -34,7 +34,7 @@
<name>Phoenix - Spark</name>
<properties>
- <spark.version>1.5.0</spark.version>
+ <spark.version>1.5.2</spark.version>
<scala.version>2.10.4</scala.version>
<scala.binary.version>2.10</scala.binary.version>
<top.dir>${project.basedir}/..</top.dir>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b8faae52/phoenix-spark/src/it/resources/setup.sql
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/it/resources/setup.sql b/phoenix-spark/src/it/resources/setup.sql
index 814d311..137d019 100644
--- a/phoenix-spark/src/it/resources/setup.sql
+++ b/phoenix-spark/src/it/resources/setup.sql
@@ -30,6 +30,13 @@ UPSERT INTO "table3" ("id", "col1") VALUES (1, 'foo')
UPSERT INTO "table3" ("id", "col1") VALUES (2, 'bar')
CREATE TABLE ARRAY_TEST_TABLE (ID BIGINT NOT NULL PRIMARY KEY, VCARRAY VARCHAR[])
UPSERT INTO ARRAY_TEST_TABLE (ID, VCARRAY) VALUES (1, ARRAY['String1', 'String2', 'String3'])
+CREATE TABLE ARRAYBUFFER_TEST_TABLE (ID BIGINT NOT NULL PRIMARY KEY, VCARRAY VARCHAR[], INTARRAY INTEGER[])
+UPSERT INTO ARRAYBUFFER_TEST_TABLE (ID, VCARRAY, INTARRAY) VALUES (1, ARRAY['String1', 'String2', 'String3'], ARRAY[1, 2, 3])
+CREATE TABLE ARRAY_ANYVAL_TEST_TABLE (ID BIGINT NOT NULL PRIMARY KEY, INTARRAY INTEGER[], BIGINTARRAY BIGINT[])
+UPSERT INTO ARRAY_ANYVAL_TEST_TABLE (ID, INTARRAY, BIGINTARRAY) VALUES (1, ARRAY[1, 2, 3], ARRAY[1, 2, 3])
+CREATE TABLE ARRAY_BYTE_TEST_TABLE (ID BIGINT NOT NULL PRIMARY KEY, BYTEARRAY TINYINT[])
+UPSERT INTO ARRAY_BYTE_TEST_TABLE (ID, BYTEARRAY) VALUES (1, ARRAY[1, 2, 3])
+CREATE TABLE VARBINARY_TEST_TABLE (ID BIGINT NOT NULL PRIMARY KEY, BIN BINARY(1), VARBIN VARBINARY, BINARRAY BINARY(1)[])
CREATE TABLE DATE_PREDICATE_TEST_TABLE (ID BIGINT NOT NULL, TIMESERIES_KEY TIMESTAMP NOT NULL CONSTRAINT pk PRIMARY KEY (ID, TIMESERIES_KEY))
UPSERT INTO DATE_PREDICATE_TEST_TABLE (ID, TIMESERIES_KEY) VALUES (1, CAST(CURRENT_TIME() AS TIMESTAMP))
CREATE TABLE OUTPUT_TEST_TABLE (id BIGINT NOT NULL PRIMARY KEY, col1 VARCHAR, col2 INTEGER, col3 DATE)
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b8faae52/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala b/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
index 1a9be6b..e6348f0 100644
--- a/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
+++ b/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
@@ -16,19 +16,17 @@ package org.apache.phoenix.spark
import java.sql.{Connection, DriverManager}
import java.util.Date
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.hbase.{HBaseConfiguration, HConstants, HBaseTestingUtility}
+import org.apache.hadoop.hbase.{HConstants}
import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT
import org.apache.phoenix.query.BaseTest
-import org.apache.phoenix.schema.{TableNotFoundException, ColumnNotFoundException}
+import org.apache.phoenix.schema.{ColumnNotFoundException}
import org.apache.phoenix.schema.types.PVarchar
import org.apache.phoenix.util.{SchemaUtil, ColumnInfo}
-import org.apache.spark.sql.{Row, SaveMode, execution, SQLContext}
+import org.apache.spark.sql.{Row, SaveMode, SQLContext}
import org.apache.spark.sql.types._
import org.apache.spark.{SparkConf, SparkContext}
import org.joda.time.DateTime
import org.scalatest._
-import org.apache.phoenix.spark._
import scala.collection.mutable.ListBuffer
@@ -470,7 +468,7 @@ class PhoenixSparkIT extends FunSuite with Matchers with BeforeAndAfterAll {
df.saveToPhoenix("TABLE2", zkUrl = Some(quorumAddress))
}
-
+
test("Ensure Dataframe supports LIKE and IN filters (PHOENIX-2328)") {
val sqlContext = new SQLContext(sc)
val df = sqlContext.load("org.apache.phoenix.spark", Map("table" -> "TABLE1",
@@ -527,4 +525,106 @@ class PhoenixSparkIT extends FunSuite with Matchers with BeforeAndAfterAll {
assert(df.select("COL1").first().getShort(0).toInt == 32767)
assert(df.select("COL2").first().getByte(0).toInt == 127)
}
+
+ test("Can save arrays from custom dataframes back to phoenix") {
+ val dataSet = List(Row(2L, Array("String1", "String2", "String3"), Array(1, 2, 3)))
+
+ val sqlContext = new SQLContext(sc)
+
+ val schema = StructType(
+ Seq(StructField("ID", LongType, nullable = false),
+ StructField("VCARRAY", ArrayType(StringType)),
+ StructField("INTARRAY", ArrayType(IntegerType))))
+
+ val rowRDD = sc.parallelize(dataSet)
+
+ // Apply the schema to the RDD.
+ val df = sqlContext.createDataFrame(rowRDD, schema)
+
+ df.write
+ .format("org.apache.phoenix.spark")
+ .options(Map("table" -> "ARRAYBUFFER_TEST_TABLE", "zkUrl" -> quorumAddress))
+ .mode(SaveMode.Overwrite)
+ .save()
+
+ // Load the results back
+ val stmt = conn.createStatement()
+ val rs = stmt.executeQuery("SELECT VCARRAY, INTARRAY FROM ARRAYBUFFER_TEST_TABLE WHERE ID = 2")
+ rs.next()
+ val stringArray = rs.getArray(1).getArray().asInstanceOf[Array[String]]
+ val intArray = rs.getArray(2).getArray().asInstanceOf[Array[Int]]
+
+ // Verify the arrays are equal
+ stringArray shouldEqual dataSet(0).getAs[Array[String]](1)
+ intArray shouldEqual dataSet(0).getAs[Array[Int]](2)
+ }
+
+ test("Can save arrays of AnyVal type back to phoenix") {
+ val dataSet = List((2L, Array(1, 2, 3), Array(1L, 2L, 3L)))
+
+ sc
+ .parallelize(dataSet)
+ .saveToPhoenix(
+ "ARRAY_ANYVAL_TEST_TABLE",
+ Seq("ID", "INTARRAY", "BIGINTARRAY"),
+ zkUrl = Some(quorumAddress)
+ )
+
+ // Load the results back
+ val stmt = conn.createStatement()
+ val rs = stmt.executeQuery("SELECT INTARRAY, BIGINTARRAY FROM ARRAY_ANYVAL_TEST_TABLE WHERE ID = 2")
+ rs.next()
+ val intArray = rs.getArray(1).getArray().asInstanceOf[Array[Int]]
+ val longArray = rs.getArray(2).getArray().asInstanceOf[Array[Long]]
+
+ // Verify the arrays are equal
+ intArray shouldEqual dataSet(0)._2
+ longArray shouldEqual dataSet(0)._3
+ }
+
+ test("Can save arrays of Byte type back to phoenix") {
+ val dataSet = List((2L, Array(1.toByte, 2.toByte, 3.toByte)))
+
+ sc
+ .parallelize(dataSet)
+ .saveToPhoenix(
+ "ARRAY_BYTE_TEST_TABLE",
+ Seq("ID", "BYTEARRAY"),
+ zkUrl = Some(quorumAddress)
+ )
+
+ // Load the results back
+ val stmt = conn.createStatement()
+ val rs = stmt.executeQuery("SELECT BYTEARRAY FROM ARRAY_BYTE_TEST_TABLE WHERE ID = 2")
+ rs.next()
+ val byteArray = rs.getArray(1).getArray().asInstanceOf[Array[Byte]]
+
+ // Verify the arrays are equal
+ byteArray shouldEqual dataSet(0)._2
+ }
+
+ test("Can save binary types back to phoenix") {
+ val dataSet = List((2L, Array[Byte](1), Array[Byte](1,2,3), Array[Array[Byte]](Array[Byte](1), Array[Byte](2))))
+
+ sc
+ .parallelize(dataSet)
+ .saveToPhoenix(
+ "VARBINARY_TEST_TABLE",
+ Seq("ID", "BIN", "VARBIN", "BINARRAY"),
+ zkUrl = Some(quorumAddress)
+ )
+
+ // Load the results back
+ val stmt = conn.createStatement()
+ val rs = stmt.executeQuery("SELECT BIN, VARBIN, BINARRAY FROM VARBINARY_TEST_TABLE WHERE ID = 2")
+ rs.next()
+ val byte = rs.getBytes("BIN")
+ val varByte = rs.getBytes("VARBIN")
+ val varByteArray = rs.getArray("BINARRAY").getArray().asInstanceOf[Array[Array[Byte]]]
+
+ // Verify the arrays are equal
+ byte shouldEqual dataSet(0)._2
+ varByte shouldEqual dataSet(0)._3
+ varByteArray shouldEqual dataSet(0)._4
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b8faae52/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRecordWritable.scala
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRecordWritable.scala b/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRecordWritable.scala
index f11f9cc..c35cc54 100644
--- a/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRecordWritable.scala
+++ b/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRecordWritable.scala
@@ -14,13 +14,11 @@
package org.apache.phoenix.spark
import java.sql.{PreparedStatement, ResultSet}
-import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapreduce.lib.db.DBWritable
-import org.apache.phoenix.schema.types.{PDataType, PDate, PhoenixArray}
+import org.apache.phoenix.schema.types._
import org.apache.phoenix.util.ColumnInfo
import org.joda.time.DateTime
-import scala.collection.{immutable, mutable}
-import scala.collection.JavaConversions._
+import scala.collection.{mutable, immutable}
class PhoenixRecordWritable(columnMetaDataList: List[ColumnInfo]) extends DBWritable {
@@ -52,16 +50,35 @@ class PhoenixRecordWritable(columnMetaDataList: List[ColumnInfo]) extends DBWrit
case _ => (v, c.getPDataType)
}
- // Save as array or object
- finalObj match {
- case obj: Array[AnyRef] => {
- // Create a java.sql.Array, need to lookup the base sql type name
- val sqlArray = statement.getConnection.createArrayOf(
- PDataType.arrayBaseType(finalType).getSqlTypeName,
- obj
- )
- statement.setArray(i + 1, sqlArray)
- }
+
+ // Helper method to create an SQL array for a specific PDatatype, and set it on the statement
+ def setArrayInStatement(obj: Array[AnyRef]): Unit = {
+ // Create a java.sql.Array, need to lookup the base sql type name
+ val sqlArray = statement.getConnection.createArrayOf(
+ PDataType.arrayBaseType(finalType).getSqlTypeName,
+ obj
+ )
+ statement.setArray(i + 1, sqlArray)
+ }
+
+ // Determine whether to save as an array or object
+ (finalObj, finalType) match {
+ case (obj: Array[AnyRef], _) => setArrayInStatement(obj)
+ case (obj: mutable.ArrayBuffer[AnyVal], _) => setArrayInStatement(obj.map(_.asInstanceOf[AnyRef]).toArray)
+ case (obj: mutable.ArrayBuffer[AnyRef], _) => setArrayInStatement(obj.toArray)
+ case (obj: mutable.WrappedArray[AnyVal], _) => setArrayInStatement(obj.map(_.asInstanceOf[AnyRef]).toArray)
+ case (obj: mutable.WrappedArray[AnyRef], _) => setArrayInStatement(obj.toArray)
+ case (obj: Array[Int], _) => setArrayInStatement(obj.map(_.asInstanceOf[AnyRef]))
+ case (obj: Array[Long], _) => setArrayInStatement(obj.map(_.asInstanceOf[AnyRef]))
+ case (obj: Array[Char], _) => setArrayInStatement(obj.map(_.asInstanceOf[AnyRef]))
+ case (obj: Array[Short], _) => setArrayInStatement(obj.map(_.asInstanceOf[AnyRef]))
+ case (obj: Array[Float], _) => setArrayInStatement(obj.map(_.asInstanceOf[AnyRef]))
+ case (obj: Array[Double], _) => setArrayInStatement(obj.map(_.asInstanceOf[AnyRef]))
+ // PVarbinary and PBinary come in as Array[Byte] but they're SQL objects
+ case (obj: Array[Byte], _ : PVarbinary) => statement.setObject(i + 1, obj)
+ case (obj: Array[Byte], _ : PBinary) => statement.setObject(i + 1, obj)
+ // Otherwise set as array type
+ case (obj: Array[Byte], _) => setArrayInStatement(obj.map(_.asInstanceOf[AnyRef]))
case _ => statement.setObject(i + 1, finalObj)
}
} else {
[21/52] [abbrv] phoenix git commit: PHOENIX-2461 Tweak shutdown of
ConnectionQueryService during testing
Posted by ma...@apache.org.
PHOENIX-2461 Tweak shutdown of ConnectionQueryService during testing
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/f237982b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/f237982b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/f237982b
Branch: refs/heads/calcite
Commit: f237982bfc0fd04c82e5aa2f340659ebdca2619e
Parents: 33ed7d9
Author: James Taylor <jt...@salesforce.com>
Authored: Sun Nov 29 14:05:37 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Sun Nov 29 14:05:37 2015 -0800
----------------------------------------------------------------------
.../test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f237982b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
index 2591886..c8928cc 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
@@ -20,6 +20,7 @@ package org.apache.phoenix.jdbc;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Properties;
+import java.util.concurrent.ThreadPoolExecutor;
import javax.annotation.concurrent.GuardedBy;
import javax.annotation.concurrent.ThreadSafe;
@@ -109,11 +110,13 @@ public class PhoenixTestDriver extends PhoenixEmbeddedDriver {
}
closed = true;
try {
- connectionQueryServices.close();
+ if (connectionQueryServices != null) connectionQueryServices.close();
} finally {
+ ThreadPoolExecutor executor = queryServices.getExecutor();
try {
queryServices.close();
} finally {
+ if (executor != null) executor.shutdown();
connectionQueryServices = null;
}
}
[14/52] [abbrv] phoenix git commit: PHOENIX-2441 Addendum to not
require transaction manager to be run
Posted by ma...@apache.org.
PHOENIX-2441 Addendum to not require transaction manager to be run
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/1b5e7fd7
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/1b5e7fd7
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/1b5e7fd7
Branch: refs/heads/calcite
Commit: 1b5e7fd71f3709f760b008f7e458c124c2868594
Parents: 104af25
Author: James Taylor <jt...@salesforce.com>
Authored: Tue Nov 24 23:19:10 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Tue Nov 24 23:19:10 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/phoenix/rpc/UpdateCacheIT.java | 15 +++++++++++++++
1 file changed, 15 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1b5e7fd7/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
index 98932e1..3b3c6c7 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
@@ -33,22 +33,29 @@ import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.util.Map;
import java.util.Properties;
import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.end2end.Shadower;
import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.MetaDataClient;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.types.PVarchar;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.TestUtil;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
+import com.google.common.collect.Maps;
+
/**
* Verifies the number of rpcs calls from {@link MetaDataClient} updateCache()
* for transactional and non-transactional tables.
@@ -57,6 +64,14 @@ public class UpdateCacheIT extends BaseHBaseManagedTimeIT {
public static final int NUM_MILLIS_IN_DAY = 86400000;
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(1);
+ props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
@Before
public void setUp() throws SQLException {
ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE);
[09/52] [abbrv] phoenix git commit: PHOENIX-1674 Snapshot isolation
transaction support through Tephra (James Taylor, Thomas D'Silva)
Posted by ma...@apache.org.
PHOENIX-1674 Snapshot isolation transaction support through Tephra (James Taylor, Thomas D'Silva)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/cc9929b5
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/cc9929b5
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/cc9929b5
Branch: refs/heads/calcite
Commit: cc9929b5c54eb979294df70929aa6eb781748b46
Parents: 76a8d18
Author: Thomas D'Silva <td...@salesforce.com>
Authored: Mon Nov 23 21:24:01 2015 -0800
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Mon Nov 23 21:24:01 2015 -0800
----------------------------------------------------------------------
bin/tephra | 310 ++++++
bin/tephra-env.sh | 55 ++
phoenix-assembly/src/build/client.xml | 4 +
phoenix-assembly/src/build/server-with-txns.xml | 48 +
.../src/build/server-without-antlr.xml | 46 +
phoenix-assembly/src/build/server.xml | 10 +-
phoenix-core/pom.xml | 24 +
.../ConnectionQueryServicesTestImpl.java | 4 +-
.../StatsCollectorWithSplitsAndMultiCFIT.java | 10 +-
.../phoenix/end2end/index/ImmutableIndexIT.java | 24 +-
.../end2end/index/IndexExpressionIT.java | 5 +-
.../apache/phoenix/end2end/index/IndexIT.java | 112 +--
.../end2end/index/MutableIndexFailureIT.java | 490 ++++++----
.../phoenix/end2end/index/MutableIndexIT.java | 68 +-
.../end2end/index/txn/MutableRollbackIT.java | 510 ++++++++++
.../phoenix/end2end/index/txn/RollbackIT.java | 167 ++++
.../end2end/index/txn/TxWriteFailureIT.java | 214 +++++
.../apache/phoenix/execute/PartialCommitIT.java | 149 +--
.../org/apache/phoenix/rpc/UpdateCacheIT.java | 108 +--
.../phoenix/rpc/UpdateCacheWithScnIT.java | 41 +
.../org/apache/phoenix/tx/TransactionIT.java | 537 +++++++++++
.../org/apache/phoenix/tx/TxCheckpointIT.java | 378 ++++++++
.../apache/phoenix/tx/TxPointInTimeQueryIT.java | 62 ++
.../phoenix/cache/IndexMetaDataCache.java | 9 +
.../apache/phoenix/cache/ServerCacheClient.java | 4 +-
.../org/apache/phoenix/cache/TenantCache.java | 2 +-
.../apache/phoenix/cache/TenantCacheImpl.java | 13 +-
.../phoenix/compile/CreateTableCompiler.java | 4 +-
.../apache/phoenix/compile/DeleteCompiler.java | 16 +-
.../apache/phoenix/compile/FromCompiler.java | 9 +-
.../apache/phoenix/compile/JoinCompiler.java | 2 +-
.../apache/phoenix/compile/PostDDLCompiler.java | 10 +-
.../phoenix/compile/StatementContext.java | 3 +-
.../compile/TupleProjectionCompiler.java | 4 +-
.../apache/phoenix/compile/UnionCompiler.java | 2 +-
.../apache/phoenix/compile/UpsertCompiler.java | 13 +-
.../coprocessor/BaseScannerRegionObserver.java | 9 +-
.../coprocessor/MetaDataEndpointImpl.java | 21 +-
.../phoenix/coprocessor/MetaDataProtocol.java | 6 +-
.../phoenix/coprocessor/ScanRegionObserver.java | 12 +-
.../coprocessor/ServerCachingEndpointImpl.java | 8 +-
.../coprocessor/ServerCachingProtocol.java | 5 +-
.../UngroupedAggregateRegionObserver.java | 6 +
.../coprocessor/generated/PTableProtos.java | 103 +-
.../generated/ServerCachingProtos.java | 120 ++-
.../phoenix/exception/SQLExceptionCode.java | 17 +
.../apache/phoenix/execute/BaseQueryPlan.java | 40 +-
.../apache/phoenix/execute/DelegateHTable.java | 300 ++++++
.../apache/phoenix/execute/HashJoinPlan.java | 27 +-
.../apache/phoenix/execute/MutationState.java | 932 ++++++++++++++-----
.../org/apache/phoenix/execute/ScanPlan.java | 2 +-
.../apache/phoenix/index/IndexMaintainer.java | 4 +-
.../phoenix/index/IndexMetaDataCacheClient.java | 12 +-
.../index/IndexMetaDataCacheFactory.java | 19 +-
.../index/PhoenixIndexFailurePolicy.java | 2 +-
.../phoenix/index/PhoenixIndexMetaData.java | 13 +
.../index/PhoenixTransactionalIndexer.java | 505 ++++++++++
.../phoenix/iterate/BaseResultIterators.java | 5 +
.../phoenix/iterate/ChunkedResultIterator.java | 25 +-
.../phoenix/iterate/ParallelIterators.java | 2 +-
.../apache/phoenix/iterate/SerialIterators.java | 2 +-
.../phoenix/iterate/TableResultIterator.java | 28 +-
.../apache/phoenix/jdbc/PhoenixConnection.java | 77 +-
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 3 +
.../org/apache/phoenix/jdbc/PhoenixDriver.java | 2 +-
.../phoenix/jdbc/PhoenixEmbeddedDriver.java | 4 +
.../apache/phoenix/jdbc/PhoenixStatement.java | 15 +-
.../apache/phoenix/join/HashCacheClient.java | 3 +-
.../apache/phoenix/join/HashCacheFactory.java | 2 +-
.../phoenix/mapreduce/PhoenixRecordReader.java | 2 +-
.../index/PhoenixIndexImportDirectMapper.java | 2 +-
.../apache/phoenix/optimize/QueryOptimizer.java | 15 +-
.../phoenix/query/ConnectionQueryServices.java | 6 +-
.../query/ConnectionQueryServicesImpl.java | 621 +++++++++---
.../query/ConnectionlessQueryServicesImpl.java | 58 +-
.../query/DelegateConnectionQueryServices.java | 27 +-
.../apache/phoenix/query/MetaDataMutated.java | 7 +-
.../apache/phoenix/query/QueryConstants.java | 23 +-
.../org/apache/phoenix/query/QueryServices.java | 3 +
.../phoenix/query/QueryServicesOptions.java | 5 +-
.../apache/phoenix/schema/DelegateTable.java | 8 +-
.../apache/phoenix/schema/MetaDataClient.java | 376 +++++---
.../org/apache/phoenix/schema/PMetaData.java | 4 +-
.../apache/phoenix/schema/PMetaDataImpl.java | 107 +--
.../java/org/apache/phoenix/schema/PTable.java | 3 +-
.../org/apache/phoenix/schema/PTableImpl.java | 73 +-
.../org/apache/phoenix/schema/PTableRef.java | 65 ++
.../apache/phoenix/schema/TableProperty.java | 5 +-
.../org/apache/phoenix/schema/TableRef.java | 12 +-
.../phoenix/trace/PhoenixMetricsSink.java | 9 +-
.../java/org/apache/phoenix/util/IndexUtil.java | 29 +-
.../org/apache/phoenix/util/MetaDataUtil.java | 7 +
.../java/org/apache/phoenix/util/ScanUtil.java | 2 +-
.../org/apache/phoenix/util/SchemaUtil.java | 29 +-
.../org/apache/phoenix/util/ServerUtil.java | 5 +-
.../apache/phoenix/util/TransactionUtil.java | 105 +++
.../phoenix/compile/QueryCompilerTest.java | 2 +-
.../phoenix/compile/ViewCompilerTest.java | 29 +-
.../phoenix/execute/CorrelatePlanTest.java | 2 +-
.../phoenix/filter/SkipScanBigFilterTest.java | 3 +-
.../covered/CoveredIndexCodecForTesting.java | 67 +-
.../phoenix/index/IndexMaintainerTest.java | 2 +-
.../apache/phoenix/jdbc/PhoenixTestDriver.java | 4 +-
.../java/org/apache/phoenix/query/BaseTest.java | 130 ++-
.../phoenix/query/ConnectionlessTest.java | 5 +-
.../phoenix/schema/PMetaDataImplTest.java | 4 +-
.../java/org/apache/phoenix/util/TestUtil.java | 38 +-
phoenix-flume/pom.xml | 7 +
phoenix-pig/pom.xml | 7 +
phoenix-protocol/src/main/PTable.proto | 1 +
.../src/main/ServerCachingService.proto | 1 +
phoenix-server/pom.xml | 7 +
pom.xml | 5 +-
113 files changed, 6349 insertions(+), 1371 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/bin/tephra
----------------------------------------------------------------------
diff --git a/bin/tephra b/bin/tephra
new file mode 100644
index 0000000..54fe301
--- /dev/null
+++ b/bin/tephra
@@ -0,0 +1,310 @@
+#!/bin/sh
+
+#
+# Copyright © 2014 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+
+# Source function library. used for "status" use case
+if [ -f "/etc/rc.d/init.d/functions" ]; then
+ PLATFORM="RHEL"
+ . /etc/rc.d/init.d/functions
+elif [ -f /lib/lsb/init-functions ] ; then
+ PLATFORM="UBUNTU"
+ . /lib/lsb/init-functions
+else
+ PLATFORM="UNSUPPORTED"
+fi
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+bin=`dirname "${BASH_SOURCE-$0}"`
+bin=`cd "$bin"; pwd`
+lib="$bin"/../lib
+conf="$bin"/../conf
+script=`basename $0`
+
+# Resolve relative symlinks
+while [ -h "$PRG" ] ; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "$PRG"`"/$link"
+ fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/.."
+APP_HOME="`pwd -P`"
+APP=`basename $0`
+
+# Load component common environment file too
+. $bin/tephra-env.sh
+
+pid=$PID_DIR/tephra-service-${IDENT_STRING}.pid
+
+# In other environment, the jars are expected to be in <HOME>/lib directory.
+# Load all the jar files. Not ideal, but we need to load only the things that
+# is needed by this script.
+if [ "$CLASSPATH" = "" ]; then
+ CLASSPATH=${lib}/*
+else
+ CLASSPATH=$CLASSPATH:${lib}/*
+fi
+
+# Load the configuration too.
+if [ -d "$conf" ]; then
+ CLASSPATH=$CLASSPATH:"$conf"/
+fi
+
+# Set Log location
+if [ ! -e $LOG_DIR ]; then
+ mkdir -p $LOG_DIR;
+fi
+export LOG_PREFIX="tephra-service-$IDENT_STRING-$HOSTNAME"
+export LOGFILE=$LOG_PREFIX.log
+loglog="${LOG_DIR}/${LOGFILE}"
+
+# set the classpath to include hadoop and hbase dependencies
+set_classpath()
+{
+ COMP_HOME=$1
+ if [ -n "$HBASE_HOME" ]; then
+ HBASE_CP=`$HBASE_HOME/bin/hbase classpath`
+ elif [ `which hbase` ]; then
+ HBASE_CP=`hbase classpath`
+ fi
+
+ export HBASE_CP
+
+ if [ -n "$HBASE_CP" ]; then
+ CP=$COMP_HOME/phoenix-assembly/target/*:$HBASE_CP:$EXTRA_CLASSPATH
+ else
+ # assume Hadoop/HBase libs are included via EXTRA_CLASSPATH
+ echo "WARN: could not find Hadoop and HBase libraries"
+ CP=$COMP_HOME/phoenix-assembly/target/*:$EXTRA_CLASSPATH
+ fi
+
+ # Setup classpaths.
+ if [ -n "$CLASSPATH" ]; then
+ CLASSPATH=$CLASSPATH:$CP
+ else
+ CLASSPATH=$CP
+ fi
+
+ export CLASSPATH
+}
+
+# Attempts to find JAVA in few ways.
+set_java ()
+{
+ # Determine the Java command to use to start the JVM.
+ if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ export JAVA="$JAVA_HOME/jre/sh/java"
+ else
+ export JAVA="$JAVA_HOME/bin/java"
+ fi
+ if [ ! -x "$JAVA" ] ; then
+ echo "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation." >&2
+ exit 1
+ fi
+ else
+ export JAVA="java"
+ which java >/dev/null 2>&1 || { echo "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation." >&2 ; exit 1; }
+ fi
+}
+
+# checks if there exists a PID that is already running. return 0 idempotently
+check_before_start()
+{
+ if [ ! -d "$PID_DIR" ]; then
+ mkdir -p "$PID_DIR"
+ fi
+ if [ -f $pid ]; then
+ if kill -0 `cat $pid` > /dev/null 2>&1; then
+ #echo "$APP $SERVICE running as process `cat $pid`. Stop it first."
+ echo "$APP running as process `cat $pid`. Stop it first."
+ exit 0
+ fi
+ fi
+}
+
+# Set Niceness
+if [ "$NICENESS" = "" ]; then
+ export NICENESS=0
+fi
+
+start() {
+ # Setup classpaths.
+ set_classpath $APP_HOME
+
+ # sets the JAVA variable.
+ set_java
+
+ check_before_start
+
+ echo "`date` Starting $APP service on `hostname`"
+ echo "`date` Starting $APP service on `hostname`" >> $loglog
+ echo "`ulimit -a`" >> $loglog 2>&1
+
+ export MAIN_CLASS="co.cask.tephra.TransactionServiceMain"
+ echo "Running class $MAIN_CLASS"
+ echo "Command: " "$JAVA" $OPTS -cp $CLASSPATH $JAVA_HEAPMAX $MAIN_CLASS >>$loglog
+ nohup nice -n $NICENESS "$JAVA" $OPTS -cp $CLASSPATH $JAVA_HEAPMAX ${MAIN_CLASS} </dev/null >>$loglog 2>&1 &
+ echo $! >$pid
+}
+
+stop() {
+ if [ -f $pid ]; then
+ pidToKill=`cat $pid`
+ # kill -0 == see if the PID exists
+ if kill -0 $pidToKill > /dev/null 2>&1; then
+ echo -n stopping $command
+ echo "`date` Terminating $command" >> $loglog
+ kill $pidToKill > /dev/null 2>&1
+ while kill -0 $pidToKill > /dev/null 2>&1;
+ do
+ echo -n "."
+ sleep 1;
+ done
+ rm $pid
+ echo
+ else
+ retval=$?
+ echo nothing to stop because kill -0 of pid $pidToKill failed with status $retval
+ fi
+ rm -f $pid
+ else
+ echo nothing to stop because no pid file $pid
+ fi
+}
+
+restart() {
+ stop
+ start
+}
+
+condrestart(){
+ case "$PLATFORM" in
+ "RHEL")
+ rh_status > /dev/null 2>&1
+ retval=$?
+ ;;
+ "UBUNTU")
+ ub_status > /dev/null 2>&1
+ retval=$?
+ ;;
+ "UNSUPPORTED")
+ echo "condrestart is not supported on platform"
+ exit 1
+ ;;
+ esac
+
+ if [[ $retval -eq 0 ]]; then
+ restart
+ fi
+}
+
+rh_status() {
+ echo "checking status"
+ # call sourced status function
+ status -p $pid
+}
+
+ub_status() {
+ echo "checking status"
+ # call sourced status function
+ status_of_proc -p $pid "$0" "$APP"
+}
+
+# Executes a specific class' main method with the classpath and environment setup
+run() {
+ classname=$1
+ shift
+ if [ -z "$classname" ]; then
+ echo "ERROR: No classname was given!"
+ echo "Usage: $0 run <fully qualified classname> [arguments]"
+ exit 1
+ fi
+ # Setup classpaths.
+ set_classpath $APP_HOME
+
+ # sets the JAVA variable.
+ set_java
+
+ echo "Running class $classname"
+ "$JAVA" $OPTS -cp $CLASSPATH $JAVA_HEAPMAX $classname $@
+}
+
+case "$1" in
+ start)
+ $1
+ ;;
+
+ stop)
+ $1
+ ;;
+
+ restart)
+ $1
+ ;;
+
+ condrestart)
+ $1
+ ;;
+
+ status)
+ case "$PLATFORM" in
+ "RHEL")
+ rh_status
+ ;;
+ "UBUNTU")
+ ub_status
+ ;;
+ "UNSUPPORTED")
+ echo "status is not supported on platform"
+ exit 1
+ ;;
+ esac
+ ;;
+
+ classpath)
+ set_classpath $APP_HOME
+ set_java
+ echo $CLASSPATH
+ ;;
+
+ run)
+ shift
+ run $@
+ ;;
+
+ *)
+ echo "Usage: $0 {start|stop|restart|status|run}"
+ exit 1
+ ;;
+
+
+esac
+exit $?
+
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/bin/tephra-env.sh
----------------------------------------------------------------------
diff --git a/bin/tephra-env.sh b/bin/tephra-env.sh
new file mode 100644
index 0000000..6906b44
--- /dev/null
+++ b/bin/tephra-env.sh
@@ -0,0 +1,55 @@
+#!/bin/sh
+
+#
+# Copyright © 2014 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+
+#
+# Common environment settings for Tephra.
+# Uncomment the lines below, where needed, and modify to adapt to your environment.
+#
+
+# A string representing this instance of the Tephra server. $USER by default.
+export IDENT_STRING=$USER
+
+# Where log files are stored. /var/log by default.
+export LOG_DIR=/tmp/tephra-$IDENT_STRING
+
+# The directory where pid files are stored. /var/run by default.
+export PID_DIR=/tmp
+
+# Add any extra classes to the classpath
+# export EXTRA_CLASSPATH
+
+# Set the JVM heap size
+# export JAVA_HEAPMAX=-Xmx2048m
+
+# Additional runtime options
+#
+# GC logging options.
+# Uncomment the following two lines, making any desired changes, to enable GC logging output
+# export GC_LOG_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:server-gc.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=2 -XX:GCLogFileSize=50M"
+# export OPTS="$OPTS $GC_LOG_OPTS"
+#
+# JMX options.
+# Uncomment the following two lines, making any desired changes, to enable remote JMX connectivity
+# export JMX_OPTS="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=13001"
+# export OPTS="$OPTS $JMX_OPTS"
+
+# Extra Java runtime options.
+# Below are what we set by default. May only work with SUN JVM.
+# For more on why as well as other possible settings,
+# see http://wiki.apache.org/hadoop/PerformanceTuning
+export OPTS="$OPTS -XX:+UseConcMarkSweepGC"
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-assembly/src/build/client.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/client.xml b/phoenix-assembly/src/build/client.xml
index 0e1e1f6..c1fe861 100644
--- a/phoenix-assembly/src/build/client.xml
+++ b/phoenix-assembly/src/build/client.xml
@@ -59,6 +59,10 @@
<include>org.apache.htrace:htrace-core</include>
<include>io.netty:netty-all</include>
<include>commons-codec:commons-codec</include>
+ <include>co.cask.tephra:tephra*</include>
+ <include>org.apache.twill:twill*</include>
+ <include>org.apache.thrift:*</include>
+ <include>com.google.code.gson:gson*</include>
</includes>
</dependencySet>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-assembly/src/build/server-with-txns.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/server-with-txns.xml b/phoenix-assembly/src/build/server-with-txns.xml
new file mode 100644
index 0000000..2808a8c
--- /dev/null
+++ b/phoenix-assembly/src/build/server-with-txns.xml
@@ -0,0 +1,48 @@
+<?xml version='1.0'?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <!-- build the phoenix server side jar, that includes phoenix-hadoopX-compat, phoenix-hadoop-compat and antlr -->
+ <id>server-with-txns</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/</outputDirectory>
+ <unpack>true</unpack>
+ <includes>
+ <include>org.apache.phoenix:phoenix-core</include>
+ <include>org.iq80.snappy:snappy</include>
+ <include>org.antlr:antlr*</include>
+ <include>co.cask.tephra:tephra*</include>
+ <include>com.google.code.gson:gson</include>
+ <include>com.google.inject.extensions:guice-assistedinject</include>
+ <include>it.unimi.dsi:fastutil</include>
+ <include>io.dropwizard.metrics:metrics-core</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-assembly/src/build/server-without-antlr.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/server-without-antlr.xml b/phoenix-assembly/src/build/server-without-antlr.xml
new file mode 100644
index 0000000..33cf0b7
--- /dev/null
+++ b/phoenix-assembly/src/build/server-without-antlr.xml
@@ -0,0 +1,46 @@
+<?xml version='1.0'?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <!-- build the phoenix server side jar, that includes phoenix-hadoopX-compat and phoenix-hadoop-compat. -->
+ <id>server-without-antlr</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/</outputDirectory>
+ <unpack>true</unpack>
+ <includes>
+ <include>org.apache.phoenix:phoenix-core</include>
+ <include>org.iq80.snappy:snappy</include>
+ <include>co.cask.tephra:tephra*</include>
+ <include>com.google.code.gson:gson</include>
+ <include>org.jruby.joni:joni</include>
+ <include>org.jruby.jcodings:jcodings</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-assembly/src/build/server.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/server.xml b/phoenix-assembly/src/build/server.xml
index 78a4b1f..141f97b 100644
--- a/phoenix-assembly/src/build/server.xml
+++ b/phoenix-assembly/src/build/server.xml
@@ -36,17 +36,13 @@
<includes>
<include>org.apache.phoenix:phoenix-core</include>
<include>org.iq80.snappy:snappy</include>
+ <include>org.antlr:antlr*</include>
+ <include>co.cask.tephra:tephra*</include>
+ <include>com.google.code.gson:gson</include>
<include>org.jruby.joni:joni</include>
<include>org.jruby.jcodings:jcodings</include>
<include>joda-time:joda-time</include>
</includes>
</dependencySet>
- <dependencySet>
- <unpack>true</unpack>
- <outputDirectory>/</outputDirectory>
- <includes>
- <include>org.antlr:antlr*</include>
- </includes>
- </dependencySet>
</dependencySets>
</assembly>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index 2bb7466..5c73ed4 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -224,6 +224,30 @@
</build>
<dependencies>
+ <!-- Transaction dependencies -->
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-api</artifactId>
+ <version>${tephra.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <version>${tephra.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <type>test-jar</type>
+ <version>${tephra.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-hbase-compat-1.1</artifactId>
+ <version>${tephra.version}</version>
+ </dependency>
+
<!-- Make sure we have all the antlr dependencies -->
<dependency>
<groupId>org.antlr</groupId>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
index bee8d21..d4f7c16 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
@@ -35,8 +35,8 @@ import org.apache.phoenix.query.QueryServices;
public class ConnectionQueryServicesTestImpl extends ConnectionQueryServicesImpl {
protected int NUM_SLAVES_BASE = 1; // number of slaves for the cluster
- public ConnectionQueryServicesTestImpl(QueryServices services, ConnectionInfo info) throws SQLException {
- super(services, info, null);
+ public ConnectionQueryServicesTestImpl(QueryServices services, ConnectionInfo info, Properties props) throws SQLException {
+ super(services, info, props);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
index 88a8215..6191a4f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
@@ -133,7 +133,7 @@ public class StatsCollectorWithSplitsAndMultiCFIT extends StatsCollectorAbstract
Collection<GuidePostsInfo> infos = TestUtil.getGuidePostsList(conn, STATS_TEST_TABLE_NAME_NEW);
long[] rowCountArr = new long[]{25, 20, 25, 25};
// CF A alone has double the bytecount because it has column qualifier A and column qualifier _0
- long[] byteCountArr = new long[]{12120, 5540, 6652, 6652};
+ long[] byteCountArr = new long[]{12144, 5540, 6652, 6652};
int i = 0;
for(GuidePostsInfo info : infos) {
assertRowCountAndByteCount(info, rowCountArr[i], byteCountArr[i]);
@@ -148,8 +148,8 @@ public class StatsCollectorWithSplitsAndMultiCFIT extends StatsCollectorAbstract
}
protected void assertRowCountAndByteCount(GuidePostsInfo info, long rowCount, long byteCount) {
- assertEquals(info.getRowCount(), rowCount);
- assertEquals(info.getByteCount(), byteCount);
+ assertEquals("Row count does not match ", rowCount, info.getRowCount());
+ assertEquals("Byte count does not match ", byteCount, info.getByteCount());
}
@Test
@@ -176,7 +176,7 @@ public class StatsCollectorWithSplitsAndMultiCFIT extends StatsCollectorAbstract
Collection<GuidePostsInfo> infos = TestUtil.getGuidePostsList(conn, STATS_TEST_TABLE_NAME);
for (GuidePostsInfo info : infos) {
assertEquals(20, info.getRowCount());
- assertEquals(11020, info.getByteCount());
+ assertEquals(11040, info.getByteCount());
break;
}
List<KeyRange> keyRanges = getAllSplits(conn, STATS_TEST_TABLE_NAME);
@@ -269,7 +269,7 @@ public class StatsCollectorWithSplitsAndMultiCFIT extends StatsCollectorAbstract
infos = TestUtil.getGuidePostsList(conn, STATS_TEST_TABLE_NAME);
for (GuidePostsInfo info : infos) {
assertEquals(20, info.getRowCount());
- assertEquals(9918, info.getByteCount());
+ assertEquals(9936, info.getByteCount());
break;
}
conn.close();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java
index 240d335..b03037c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java
@@ -48,17 +48,28 @@ public class ImmutableIndexIT extends BaseHBaseManagedTimeIT {
private final boolean localIndex;
private final String tableDDLOptions;
+ private final String tableName;
+ private final String indexName;
+ private final String fullTableName;
+ private final String fullIndexName;
- public ImmutableIndexIT(boolean localIndex) {
+ public ImmutableIndexIT(boolean localIndex, boolean transactional) {
this.localIndex = localIndex;
StringBuilder optionBuilder = new StringBuilder("IMMUTABLE_ROWS=true");
+ if (transactional) {
+ optionBuilder.append(", TRANSACTIONAL=true");
+ }
this.tableDDLOptions = optionBuilder.toString();
+ this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + ( transactional ? "_TXN" : "");
+ this.indexName = "IDX" + ( transactional ? "_TXN" : "");
+ this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ this.fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
}
- @Parameters(name="localIndex = {0}")
+ @Parameters(name="localIndex = {0} , transactional = {1}")
public static Collection<Boolean[]> data() {
return Arrays.asList(new Boolean[][] {
- { false}, { true }
+ { false, false }, { false, true }, { true, false }, { true, true }
});
}
@@ -67,11 +78,6 @@ public class ImmutableIndexIT extends BaseHBaseManagedTimeIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
Statement stmt = conn.createStatement();
stmt.execute(ddl);
@@ -102,4 +108,4 @@ public class ImmutableIndexIT extends BaseHBaseManagedTimeIT {
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
index e3d1db7..7da0d85 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
@@ -38,6 +38,7 @@ import org.apache.phoenix.util.DateUtil;
import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.TestUtil;
import org.junit.Test;
public class IndexExpressionIT extends BaseHBaseManagedTimeIT {
@@ -75,7 +76,7 @@ public class IndexExpressionIT extends BaseHBaseManagedTimeIT {
stmt.setInt(3, i);
stmt.setLong(4, i);
stmt.setBigDecimal(5, new BigDecimal(i*0.5d));
- Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * NUM_MILLIS_IN_DAY);
+ Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * TestUtil.NUM_MILLIS_IN_DAY);
stmt.setDate(6, date);
stmt.setString(7, "a.varchar" + String.valueOf(i));
stmt.setString(8, "a.char" + String.valueOf(i));
@@ -98,7 +99,7 @@ public class IndexExpressionIT extends BaseHBaseManagedTimeIT {
+ "_A.VARCHAR" + String.valueOf(i) + "_" + StringUtils.rightPad("B.CHAR" + String.valueOf(i), 10, ' '),
rs.getString(1));
assertEquals(i * 3, rs.getInt(2));
- Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i) * NUM_MILLIS_IN_DAY);
+ Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i) * TestUtil.NUM_MILLIS_IN_DAY);
assertEquals(date, rs.getDate(3));
assertEquals(date, rs.getDate(4));
assertEquals(date, rs.getDate(5));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
index 695225f..d2c6267 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
@@ -33,7 +33,6 @@ import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -44,20 +43,33 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
private final boolean localIndex;
private final String tableDDLOptions;
+ private final String tableName;
+ private final String indexName;
+ private final String fullTableName;
+ private final String fullIndexName;
- public IndexIT(boolean localIndex, boolean mutable) {
+ public IndexIT(boolean localIndex, boolean mutable, boolean transactional) {
this.localIndex = localIndex;
StringBuilder optionBuilder = new StringBuilder();
if (!mutable)
optionBuilder.append(" IMMUTABLE_ROWS=true ");
+ if (transactional) {
+ if (!(optionBuilder.length()==0))
+ optionBuilder.append(",");
+ optionBuilder.append(" TRANSACTIONAL=true ");
+ }
this.tableDDLOptions = optionBuilder.toString();
+ this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + ( transactional ? "_TXN" : "");
+ this.indexName = "IDX" + ( transactional ? "_TXN" : "");
+ this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ this.fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
}
- @Parameters(name="localIndex = {0} , mutable = {1}")
+ @Parameters(name="localIndex = {0} , mutable = {1} , transactional = {2}")
public static Collection<Boolean[]> data() {
return Arrays.asList(new Boolean[][] {
- { false, false }, { false, true },
- { true, false }, { true, true }
+ { false, false, false }, { false, false, true }, { false, true, false }, { false, true, true },
+ { true, false, false }, { true, false, true }, { true, true, false }, { true, true, true }
});
}
@@ -66,10 +78,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
Statement stmt = conn.createStatement();
stmt.execute(ddl);
@@ -122,16 +130,10 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
}
@Test
- @Ignore("Failing due to zero byte incorrectly being stripped from row key") // FIXME: fixed in master, so remove this ignore tag when merged.
public void testDeleteFromAllPKColumnIndex() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
Statement stmt = conn.createStatement();
stmt.execute(ddl);
@@ -185,11 +187,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
@Test
public void testDeleteFromNonPKColumnIndex() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
@@ -242,10 +239,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
Statement stmt = conn.createStatement();
stmt.execute(ddl);
@@ -264,10 +257,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
Statement stmt = conn.createStatement();
stmt.execute(ddl);
@@ -291,10 +280,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
String ddl ="CREATE TABLE " + fullTableName + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions;
Statement stmt = conn.createStatement();
stmt.execute(ddl);
@@ -319,11 +304,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
String ddl ="CREATE TABLE " + fullTableName
+ " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) DEFAULT_COLUMN_FAMILY='A'" + (!tableDDLOptions.isEmpty() ? "," + tableDDLOptions : "");
Statement stmt = conn.createStatement();
@@ -361,11 +341,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
Date date = new Date(System.currentTimeMillis());
createMultiCFTestTable(fullTableName, tableDDLOptions);
@@ -423,12 +398,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
String ddl = "CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) " + tableDDLOptions;
conn.createStatement().execute(ddl);
query = "SELECT * FROM " + fullTableName;
@@ -504,12 +473,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
String ddl = "CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, a.v1 VARCHAR, a.v2 VARCHAR, b.v1 VARCHAR) " + tableDDLOptions;
conn.createStatement().execute(ddl);
query = "SELECT * FROM " + fullTableName;
@@ -567,12 +530,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
// make sure that the tables are empty, but reachable
conn.createStatement().execute(
"CREATE TABLE " + fullTableName
@@ -629,34 +586,29 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
@Test
public void testMultipleUpdatesAcrossRegions() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String testTable = fullTableName+"_MULTIPLE_UPDATES";
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
// make sure that the tables are empty, but reachable
conn.createStatement().execute(
- "CREATE TABLE " + fullTableName
+ "CREATE TABLE " + testTable
+ " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) " + HTableDescriptor.MAX_FILESIZE + "=1, " + HTableDescriptor.MEMSTORE_FLUSHSIZE + "=1 "
+ (!tableDDLOptions.isEmpty() ? "," + tableDDLOptions : "") + "SPLIT ON ('b')");
- query = "SELECT * FROM " + fullTableName;
+ query = "SELECT * FROM " + testTable;
rs = conn.createStatement().executeQuery(query);
assertFalse(rs.next());
conn.createStatement().execute(
- "CREATE " + (localIndex ? "LOCAL " : "") + "INDEX " + indexName + " ON " + fullTableName + " (v1, v2)");
+ "CREATE " + (localIndex ? "LOCAL " : "") + "INDEX " + indexName + " ON " + testTable + " (v1, v2)");
query = "SELECT * FROM " + fullIndexName;
rs = conn.createStatement().executeQuery(query);
assertFalse(rs.next());
// load some data into the table
PreparedStatement stmt =
- conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
+ conn.prepareStatement("UPSERT INTO " + testTable + " VALUES(?,?,?)");
stmt.setString(1, "a");
stmt.setString(2, "x");
stmt.setString(3, "1");
@@ -688,10 +640,10 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
assertEquals("c", rs.getString(3));
assertFalse(rs.next());
- query = "SELECT * FROM " + fullTableName;
+ query = "SELECT * FROM " + testTable;
rs = conn.createStatement().executeQuery("EXPLAIN " + query);
if (localIndex) {
- assertEquals("CLIENT PARALLEL 2-WAY RANGE SCAN OVER _LOCAL_IDX_" + fullTableName+" [-32768]\n"
+ assertEquals("CLIENT PARALLEL 2-WAY RANGE SCAN OVER _LOCAL_IDX_" + testTable+" [-32768]\n"
+ " SERVER FILTER BY FIRST KEY ONLY\n"
+ "CLIENT MERGE SORT",
QueryUtil.getExplainPlan(rs));
@@ -726,12 +678,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
-
conn.createStatement().execute("CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, \"V1\" VARCHAR, \"v2\" VARCHAR)"+tableDDLOptions);
query = "SELECT * FROM "+fullTableName;
rs = conn.createStatement().executeQuery(query);
@@ -815,11 +761,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
-
String ddl = "CREATE TABLE " + fullTableName +" (PK1 CHAR(2) NOT NULL PRIMARY KEY, CF1.COL1 BIGINT) " + tableDDLOptions;
conn.createStatement().execute(ddl);
ddl = "CREATE " + (localIndex ? "LOCAL " : "") + "INDEX " + indexName + " ON " + fullTableName + "(COL1)";
@@ -838,11 +779,6 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
conn.setAutoCommit(false);
String query;
ResultSet rs;
- // create unique table and index names for each parameterized test
- String tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + "_" + System.currentTimeMillis();
- String indexName = "IDX" + "_" + System.currentTimeMillis();
- String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
- String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
Date date = new Date(System.currentTimeMillis());
createMultiCFTestTable(fullTableName, tableDDLOptions);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index 364b358..39c90be 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -26,11 +26,13 @@ import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Properties;
@@ -69,11 +71,15 @@ import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.StringUtil;
+import org.apache.phoenix.util.TestUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
/**
*
* Test for failure of region server to write to index table.
@@ -85,18 +91,30 @@ import org.junit.experimental.categories.Category;
*/
@Category(NeedsOwnMiniClusterTest.class)
+@RunWith(Parameterized.class)
public class MutableIndexFailureIT extends BaseTest {
private static final int NUM_SLAVES = 4;
- private static String url;
private static PhoenixTestDriver driver;
private static HBaseTestingUtility util;
private Timer scheduleTimer;
- private static final String SCHEMA_NAME = "S";
- private static final String INDEX_TABLE_NAME = "I";
- private static final String DATA_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, "T");
- private static final String INDEX_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, "I");
+ private String tableName;
+ private String indexName;
+ private String fullTableName;
+ private String fullIndexName;
+ private boolean transactional;
+ private final String tableDDLOptions;
+
+ public MutableIndexFailureIT(boolean transactional) {
+ this.transactional = transactional;
+ this.tableDDLOptions = transactional ? " TRANSACTIONAL=true " : "";
+ this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME;
+ this.indexName = "IDX";
+ this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ this.fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
+ }
+
@Before
public void doSetup() throws Exception {
Configuration conf = HBaseConfiguration.create();
@@ -114,8 +132,15 @@ public class MutableIndexFailureIT extends BaseTest {
url = JDBC_PROTOCOL + JDBC_PROTOCOL_SEPARATOR + LOCALHOST + JDBC_PROTOCOL_SEPARATOR + clientPort
+ JDBC_PROTOCOL_TERMINATOR + PHOENIX_TEST_DRIVER_URL_PARAM;
driver = initAndRegisterDriver(url, ReadOnlyProps.EMPTY_PROPS);
+ clusterInitialized = true;
+ setupTxManager();
}
-
+
+ @Parameters(name = "transactional = {0}")
+ public static Collection<Boolean[]> data() {
+ return Arrays.asList(new Boolean[][] { { false }, { true } });
+ }
+
@After
public void tearDown() throws Exception {
try {
@@ -135,230 +160,279 @@ public class MutableIndexFailureIT extends BaseTest {
@Ignore("See PHOENIX-2331")
@Test(timeout=300000)
public void testWriteFailureDisablesLocalIndex() throws Exception {
- testWriteFailureDisablesIndex(true);
+ helpTestWriteFailureDisablesIndex(true);
}
@Ignore("See PHOENIX-2332")
@Test(timeout=300000)
public void testWriteFailureDisablesIndex() throws Exception {
- testWriteFailureDisablesIndex(false);
+ helpTestWriteFailureDisablesIndex(false);
}
- public void testWriteFailureDisablesIndex(boolean localIndex) throws Exception {
- String query;
- ResultSet rs;
-
+ public void helpTestWriteFailureDisablesIndex(boolean localIndex) throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = driver.connect(url, props);
- conn.setAutoCommit(false);
- conn.createStatement().execute(
- "CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
- query = "SELECT * FROM " + DATA_TABLE_FULL_NAME;
- rs = conn.createStatement().executeQuery(query);
- assertFalse(rs.next());
-
- if(localIndex) {
+ try (Connection conn = driver.connect(url, props)) {
+ String query;
+ ResultSet rs;
+ conn.setAutoCommit(false);
conn.createStatement().execute(
- "CREATE LOCAL INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v1) INCLUDE (v2)");
- conn.createStatement().execute(
- "CREATE LOCAL INDEX " + INDEX_TABLE_NAME+ "_2" + " ON " + DATA_TABLE_FULL_NAME + " (v2) INCLUDE (v1)");
- } else {
- conn.createStatement().execute(
- "CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v1) INCLUDE (v2)");
- }
+ "CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) "+tableDDLOptions);
+ query = "SELECT * FROM " + fullTableName;
+ rs = conn.createStatement().executeQuery(query);
+ assertFalse(rs.next());
+
+ if(localIndex) {
+ conn.createStatement().execute(
+ "CREATE LOCAL INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE (v2)");
+ conn.createStatement().execute(
+ "CREATE LOCAL INDEX " + indexName+ "_2" + " ON " + fullTableName + " (v2) INCLUDE (v1)");
+ } else {
+ conn.createStatement().execute(
+ "CREATE INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE (v2)");
+ }
+
+ query = "SELECT * FROM " + fullIndexName;
+ rs = conn.createStatement().executeQuery(query);
+ assertFalse(rs.next());
+
+ // Verify the metadata for index is correct.
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
+ new String[] { PTableType.INDEX.toString() });
+ assertTrue(rs.next());
+ assertEquals(indexName, rs.getString(3));
+ assertEquals(PIndexState.ACTIVE.toString(), rs.getString("INDEX_STATE"));
+ assertFalse(rs.next());
- query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME;
- rs = conn.createStatement().executeQuery(query);
- assertFalse(rs.next());
-
- // Verify the metadata for index is correct.
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(SCHEMA_NAME), INDEX_TABLE_NAME,
- new String[] { PTableType.INDEX.toString() });
- assertTrue(rs.next());
- assertEquals(INDEX_TABLE_NAME, rs.getString(3));
- assertEquals(PIndexState.ACTIVE.toString(), rs.getString("INDEX_STATE"));
- assertFalse(rs.next());
-
- PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)");
- stmt.setString(1, "a");
- stmt.setString(2, "x");
- stmt.setString(3, "1");
- stmt.execute();
- conn.commit();
-
- TableName indexTable =
- TableName.valueOf(localIndex ? MetaDataUtil
- .getLocalIndexTableName(DATA_TABLE_FULL_NAME) : INDEX_TABLE_FULL_NAME);
- HBaseAdmin admin = this.util.getHBaseAdmin();
- HTableDescriptor indexTableDesc = admin.getTableDescriptor(indexTable);
- try{
- admin.disableTable(indexTable);
- admin.deleteTable(indexTable);
- } catch (TableNotFoundException ignore) {}
-
- stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)");
- stmt.setString(1, "a2");
- stmt.setString(2, "x2");
- stmt.setString(3, "2");
- stmt.execute();
- try {
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
+ stmt.setString(1, "a");
+ stmt.setString(2, "x");
+ stmt.setString(3, "1");
+ stmt.execute();
conn.commit();
- } catch (SQLException e) {}
-
- // Verify the metadata for index is correct.
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(SCHEMA_NAME), INDEX_TABLE_NAME,
- new String[] { PTableType.INDEX.toString() });
- assertTrue(rs.next());
- assertEquals(INDEX_TABLE_NAME, rs.getString(3));
- assertEquals(PIndexState.DISABLE.toString(), rs.getString("INDEX_STATE"));
- assertFalse(rs.next());
- if(localIndex) {
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(SCHEMA_NAME), INDEX_TABLE_NAME+"_2",
- new String[] { PTableType.INDEX.toString() });
+
+ TableName indexTable =
+ TableName.valueOf(localIndex ? MetaDataUtil
+ .getLocalIndexTableName(fullTableName) : fullIndexName);
+ HBaseAdmin admin = this.util.getHBaseAdmin();
+ HTableDescriptor indexTableDesc = admin.getTableDescriptor(indexTable);
+ try{
+ admin.disableTable(indexTable);
+ admin.deleteTable(indexTable);
+ } catch (TableNotFoundException ignore) {}
+
+ stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
+ stmt.setString(1, "a2");
+ stmt.setString(2, "x2");
+ stmt.setString(3, "2");
+ stmt.execute();
+ if (transactional) {
+ try {
+ conn.commit();
+ fail();
+ } catch (SQLException e) {
+ conn.rollback();
+ }
+ }
+ else {
+ conn.commit();
+ }
+
+ // Verify the metadata for index is correct.
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
+ new String[] { PTableType.INDEX.toString() });
assertTrue(rs.next());
- assertEquals(INDEX_TABLE_NAME+"_2", rs.getString(3));
- assertEquals(PIndexState.DISABLE.toString(), rs.getString("INDEX_STATE"));
+ assertEquals(indexName, rs.getString(3));
+ // the index is only disabled for non-txn tables upon index table write failure
+ PIndexState indexState = transactional ? PIndexState.ACTIVE : PIndexState.DISABLE;
+ assertEquals(indexState.toString(), rs.getString("INDEX_STATE"));
assertFalse(rs.next());
- }
-
- // Verify UPSERT on data table still work after index is disabled
- stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)");
- stmt.setString(1, "a3");
- stmt.setString(2, "x3");
- stmt.setString(3, "3");
- stmt.execute();
- conn.commit();
-
- query = "SELECT v2 FROM " + DATA_TABLE_FULL_NAME + " where v1='x3'";
- rs = conn.createStatement().executeQuery("EXPLAIN " + query);
- assertTrue(QueryUtil.getExplainPlan(rs).contains("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + DATA_TABLE_FULL_NAME));
- rs = conn.createStatement().executeQuery(query);
- assertTrue(rs.next());
-
- // recreate index table
- admin.createTable(indexTableDesc);
- do {
- Thread.sleep(15 * 1000); // sleep 15 secs
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(SCHEMA_NAME), INDEX_TABLE_NAME,
- new String[] { PTableType.INDEX.toString() });
- assertTrue(rs.next());
- if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
- break;
- }
- if(localIndex) {
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(SCHEMA_NAME), INDEX_TABLE_NAME+"_2",
+ if(localIndex) {
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName + "_2",
+ new String[] { PTableType.INDEX.toString() });
+ assertTrue(rs.next());
+ assertEquals(indexName + "_2", rs.getString(3));
+ assertEquals(indexState.toString(), rs.getString("INDEX_STATE"));
+ assertFalse(rs.next());
+ }
+
+ // if the table is transactional the write to the index table will fail because the
+ // index has not been disabled
+ if (!transactional) {
+ // Verify UPSERT on data table still work after index is disabled
+ stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
+ stmt.setString(1, "a3");
+ stmt.setString(2, "x3");
+ stmt.setString(3, "3");
+ stmt.execute();
+ conn.commit();
+ }
+
+ if (transactional) {
+ // if the table was transactional there should be 1 row (written before the index
+ // was disabled)
+ query = "SELECT /*+ NO_INDEX */ v2 FROM " + fullTableName;
+ rs = conn.createStatement().executeQuery("EXPLAIN " + query);
+ String expectedPlan =
+ "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + fullTableName;
+ assertEquals(expectedPlan, QueryUtil.getExplainPlan(rs));
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("1", rs.getString(1));
+ assertFalse(rs.next());
+ } else {
+ // if the table was not transactional there should be three rows (all writes to data
+ // table should succeed)
+ query = "SELECT v2 FROM " + fullTableName;
+ rs = conn.createStatement().executeQuery("EXPLAIN " + query);
+ String expectedPlan =
+ "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + fullTableName;
+ assertEquals(expectedPlan, QueryUtil.getExplainPlan(rs));
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("1", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("2", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("3", rs.getString(1));
+ assertFalse(rs.next());
+ }
+
+ // recreate index table
+ admin.createTable(indexTableDesc);
+ do {
+ Thread.sleep(15 * 1000); // sleep 15 secs
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
new String[] { PTableType.INDEX.toString() });
assertTrue(rs.next());
if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
break;
}
- }
- } while(true);
-
- // verify index table has data
- query = "SELECT count(1) FROM " + INDEX_TABLE_FULL_NAME;
- rs = conn.createStatement().executeQuery(query);
- assertTrue(rs.next());
-
- // using 2 here because we only partially build index from where we failed and the oldest
- // index row has been deleted when we dropped the index table during test.
- assertEquals(2, rs.getInt(1));
- }
-
- @Test(timeout=300000)
- public void testWriteFailureWithRegionServerDown() throws Exception {
- String query;
- ResultSet rs;
-
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = driver.connect(url, props);
- conn.setAutoCommit(false);
- conn.createStatement().execute(
- "CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
- query = "SELECT * FROM " + DATA_TABLE_FULL_NAME;
- rs = conn.createStatement().executeQuery(query);
- assertFalse(rs.next());
+ if(localIndex) {
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName + "_2",
+ new String[] { PTableType.INDEX.toString() });
+ assertTrue(rs.next());
+ if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
+ break;
+ }
+ }
+ } while(true);
+
+ // Verify UPSERT on data table still work after index table is recreated
+ stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
+ stmt.setString(1, "a4");
+ stmt.setString(2, "x4");
+ stmt.setString(3, "4");
+ stmt.execute();
+ conn.commit();
- conn.createStatement().execute(
- "CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v1) INCLUDE (v2)");
- query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME;
- rs = conn.createStatement().executeQuery(query);
- assertFalse(rs.next());
+ // verify index table has data
+ query = "SELECT count(1) FROM " + fullIndexName;
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
- // Verify the metadata for index is correct.
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(SCHEMA_NAME), INDEX_TABLE_NAME,
- new String[] { PTableType.INDEX.toString() });
- assertTrue(rs.next());
- assertEquals(INDEX_TABLE_NAME, rs.getString(3));
- assertEquals(PIndexState.ACTIVE.toString(), rs.getString("INDEX_STATE"));
- assertFalse(rs.next());
-
- PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)");
- stmt.setString(1, "a");
- stmt.setString(2, "x");
- stmt.setString(3, "1");
- stmt.execute();
- conn.commit();
-
- // find a RS which doesn't has CATALOG table
- TableName catalogTable = TableName.valueOf("SYSTEM.CATALOG");
- TableName indexTable = TableName.valueOf(INDEX_TABLE_FULL_NAME);
- final HBaseCluster cluster = this.util.getHBaseCluster();
- Collection<ServerName> rss = cluster.getClusterStatus().getServers();
- HBaseAdmin admin = this.util.getHBaseAdmin();
- List<HRegionInfo> regions = admin.getTableRegions(catalogTable);
- ServerName catalogRS = cluster.getServerHoldingRegion(regions.get(0).getTable(),
- regions.get(0).getRegionName());
- ServerName metaRS = cluster.getServerHoldingMeta();
- ServerName rsToBeKilled = null;
-
- // find first RS isn't holding META or CATALOG table
- for(ServerName curRS : rss) {
- if(!curRS.equals(catalogRS) && !metaRS.equals(curRS)) {
- rsToBeKilled = curRS;
- break;
+ // for txn tables there will be only one row in the index (a4)
+ // for non txn tables there will be three rows because we only partially build index
+ // from where we failed and the oldest
+ // index row has been deleted when we dropped the index table during test
+ assertEquals(transactional ? 1 : 3, rs.getInt(1));
}
}
- assertTrue(rsToBeKilled != null);
-
- regions = admin.getTableRegions(indexTable);
- final HRegionInfo indexRegion = regions.get(0);
- final ServerName dstRS = rsToBeKilled;
- admin.move(indexRegion.getEncodedNameAsBytes(), Bytes.toBytes(rsToBeKilled.getServerName()));
- this.util.waitFor(30000, 200, new Waiter.Predicate<Exception>() {
- @Override
- public boolean evaluate() throws Exception {
- ServerName sn = cluster.getServerHoldingRegion(indexRegion.getTable(),
- indexRegion.getRegionName());
- return (sn != null && sn.equals(dstRS));
- }
- });
-
- // use timer sending updates in every 10ms
- this.scheduleTimer = new Timer(true);
- this.scheduleTimer.schedule(new SendingUpdatesScheduleTask(conn), 0, 10);
- // let timer sending some updates
- Thread.sleep(100);
- // kill RS hosting index table
- this.util.getHBaseCluster().killRegionServer(rsToBeKilled);
-
- // wait for index table completes recovery
- this.util.waitUntilAllRegionsAssigned(indexTable);
+ @Test(timeout=300000)
+ public void testWriteFailureWithRegionServerDown() throws Exception {
+ String query;
+ ResultSet rs;
+
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = driver.connect(url, props);) {
+ conn.setAutoCommit(false);
+ conn.createStatement().execute(
+ "CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) "+tableDDLOptions);
+ query = "SELECT * FROM " + fullTableName;
+ rs = conn.createStatement().executeQuery(query);
+ assertFalse(rs.next());
- // Verify the metadata for index is correct.
- do {
- Thread.sleep(15 * 1000); // sleep 15 secs
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(SCHEMA_NAME), INDEX_TABLE_NAME,
- new String[] { PTableType.INDEX.toString() });
- assertTrue(rs.next());
- if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
- break;
- }
- } while(true);
- this.scheduleTimer.cancel();
+ conn.createStatement().execute(
+ "CREATE INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE (v2)");
+ query = "SELECT * FROM " + fullIndexName;
+ rs = conn.createStatement().executeQuery(query);
+ assertFalse(rs.next());
- assertEquals(cluster.getClusterStatus().getDeadServers(), 1);
+ // Verify the metadata for index is correct.
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
+ new String[] { PTableType.INDEX.toString() });
+ assertTrue(rs.next());
+ assertEquals(indexName, rs.getString(3));
+ assertEquals(PIndexState.ACTIVE.toString(), rs.getString("INDEX_STATE"));
+ assertFalse(rs.next());
+
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
+ stmt.setString(1, "a");
+ stmt.setString(2, "x");
+ stmt.setString(3, "1");
+ stmt.execute();
+ conn.commit();
+
+ // find a RS which doesn't has CATALOG table
+ TableName catalogTable = TableName.valueOf("SYSTEM.CATALOG");
+ TableName indexTable = TableName.valueOf(fullIndexName);
+ final HBaseCluster cluster = this.util.getHBaseCluster();
+ Collection<ServerName> rss = cluster.getClusterStatus().getServers();
+ HBaseAdmin admin = this.util.getHBaseAdmin();
+ List<HRegionInfo> regions = admin.getTableRegions(catalogTable);
+ ServerName catalogRS = cluster.getServerHoldingRegion(regions.get(0).getTable(),
+ regions.get(0).getRegionName());
+ ServerName metaRS = cluster.getServerHoldingMeta();
+ ServerName rsToBeKilled = null;
+
+ // find first RS isn't holding META or CATALOG table
+ for(ServerName curRS : rss) {
+ if(!curRS.equals(catalogRS) && !metaRS.equals(curRS)) {
+ rsToBeKilled = curRS;
+ break;
+ }
+ }
+ assertTrue(rsToBeKilled != null);
+
+ regions = admin.getTableRegions(indexTable);
+ final HRegionInfo indexRegion = regions.get(0);
+ final ServerName dstRS = rsToBeKilled;
+ admin.move(indexRegion.getEncodedNameAsBytes(), Bytes.toBytes(rsToBeKilled.getServerName()));
+ this.util.waitFor(30000, 200, new Waiter.Predicate<Exception>() {
+ @Override
+ public boolean evaluate() throws Exception {
+ ServerName sn = cluster.getServerHoldingRegion(indexRegion.getTable(),
+ indexRegion.getRegionName());
+ return (sn != null && sn.equals(dstRS));
+ }
+ });
+
+ // use timer sending updates in every 10ms
+ this.scheduleTimer = new Timer(true);
+ this.scheduleTimer.schedule(new SendingUpdatesScheduleTask(conn, fullTableName), 0, 10);
+ // let timer sending some updates
+ Thread.sleep(100);
+
+ // kill RS hosting index table
+ this.util.getHBaseCluster().killRegionServer(rsToBeKilled);
+
+ // wait for index table completes recovery
+ this.util.waitUntilAllRegionsAssigned(indexTable);
+
+ // Verify the metadata for index is correct.
+ do {
+ Thread.sleep(15 * 1000); // sleep 15 secs
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
+ new String[] { PTableType.INDEX.toString() });
+ assertTrue(rs.next());
+ if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
+ break;
+ }
+ } while(true);
+ this.scheduleTimer.cancel();
+
+ assertEquals(cluster.getClusterStatus().getDeadServers(), 1);
+ }
}
static class SendingUpdatesScheduleTask extends TimerTask {
@@ -368,10 +442,12 @@ public class MutableIndexFailureIT extends BaseTest {
// running
private final static AtomicInteger inProgress = new AtomicInteger(0);
private final Connection conn;
+ private final String fullTableName;
private int inserts = 0;
- public SendingUpdatesScheduleTask(Connection conn) {
+ public SendingUpdatesScheduleTask(Connection conn, String fullTableName) {
this.conn = conn;
+ this.fullTableName = fullTableName;
}
public void run() {
@@ -382,7 +458,7 @@ public class MutableIndexFailureIT extends BaseTest {
try {
inProgress.incrementAndGet();
inserts++;
- PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)");
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
stmt.setString(1, "a" + inserts);
stmt.setString(2, "x" + inserts);
stmt.setString(3, String.valueOf(inserts));
[50/52] [abbrv] phoenix git commit: PHOENIX-2506 UPDATE STATS could
end up updating incomplete/incorrect stats in case of failures
Posted by ma...@apache.org.
PHOENIX-2506 UPDATE STATS could end up updating incomplete/incorrect stats in case of failures
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/24a94a25
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/24a94a25
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/24a94a25
Branch: refs/heads/calcite
Commit: 24a94a25717f865ef382c88ad8986c62c2599f57
Parents: c2a1f74
Author: Samarth <sa...@apache.org>
Authored: Wed Dec 9 16:35:28 2015 -0800
Committer: Samarth <sa...@apache.org>
Committed: Wed Dec 9 16:35:28 2015 -0800
----------------------------------------------------------------------
.../phoenix/coprocessor/UngroupedAggregateRegionObserver.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/24a94a25/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index ffdc05f..01a5e38 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -278,6 +278,8 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
long rowCount = 0;
final RegionScanner innerScanner = theScanner;
region.startRegionOperation();
+ boolean updateStats = stats != null;
+ boolean success = false;
try {
synchronized (innerScanner) {
do {
@@ -286,7 +288,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
// since this is an indication of whether or not there are more values after the
// ones returned
hasMore = innerScanner.nextRaw(results);
- if (stats != null) {
+ if (updateStats) {
stats.collectStatistics(results);
}
if (!results.isEmpty()) {
@@ -516,10 +518,11 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
hasAny = true;
}
} while (hasMore);
+ success = true;
}
} finally {
try {
- if (stats != null) {
+ if (success && updateStats) {
try {
stats.updateStatistic(region);
} finally {
[23/52] [abbrv] phoenix git commit: Revert "PHOENIX-2462 Add tephra
test dependency to spark module"
Posted by ma...@apache.org.
Revert "PHOENIX-2462 Add tephra test dependency to spark module"
This reverts commit 0e45e24581e592a21745cf07b8c7620bbd1f3263.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d60713e9
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d60713e9
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d60713e9
Branch: refs/heads/calcite
Commit: d60713e96f2c63b2a20551b7b1e2c05791882706
Parents: 0e45e24
Author: James Taylor <jt...@salesforce.com>
Authored: Sun Nov 29 18:30:37 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Sun Nov 29 18:30:37 2015 -0800
----------------------------------------------------------------------
phoenix-spark/pom.xml | 7 -------
1 file changed, 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d60713e9/phoenix-spark/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-spark/pom.xml b/phoenix-spark/pom.xml
index c0ad626..5edd5be 100644
--- a/phoenix-spark/pom.xml
+++ b/phoenix-spark/pom.xml
@@ -478,13 +478,6 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>co.cask.tephra</groupId>
- <artifactId>tephra-core</artifactId>
- <type>test-jar</type>
- <version>${tephra.version}</version>
- <scope>test</scope>
- </dependency>
</dependencies>
<build>
[39/52] [abbrv] phoenix git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/phoenix
Posted by ma...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/phoenix
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/1c2b9b0e
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/1c2b9b0e
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/1c2b9b0e
Branch: refs/heads/calcite
Commit: 1c2b9b0e70876862f6be96749cdc8e1522a025fc
Parents: bc9974b 26a6f88
Author: maryannxue <we...@intel.com>
Authored: Wed Dec 2 21:12:29 2015 -0500
Committer: maryannxue <we...@intel.com>
Committed: Wed Dec 2 21:12:29 2015 -0500
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/index/MutableIndexFailureIT.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
[06/52] [abbrv] phoenix git commit: PHOENIX-1674 Snapshot isolation
transaction support through Tephra (James Taylor, Thomas D'Silva)
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java
index 54c688a..65a43de 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java
@@ -26,7 +26,7 @@ import java.sql.SQLException;
import java.util.List;
import java.util.Set;
-import com.google.common.collect.Sets;
+import co.cask.tephra.Transaction;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.io.WritableUtils;
import org.apache.phoenix.cache.GlobalCache;
import org.apache.phoenix.cache.TenantCache;
+import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.execute.TupleProjector;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.expression.KeyValueColumnExpression;
@@ -58,8 +59,10 @@ import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.ServerUtil;
+import org.apache.phoenix.util.TransactionUtil;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
@@ -191,6 +194,7 @@ public class ScanRegionObserver extends BaseScannerRegionObserver {
Region dataRegion = null;
IndexMaintainer indexMaintainer = null;
byte[][] viewConstants = null;
+ Transaction tx = null;
ColumnReference[] dataColumns = IndexUtil.deserializeDataTableColumnsToJoin(scan);
if (dataColumns != null) {
tupleProjector = IndexUtil.getTupleProjector(scan, dataColumns);
@@ -199,14 +203,16 @@ public class ScanRegionObserver extends BaseScannerRegionObserver {
List<IndexMaintainer> indexMaintainers = localIndexBytes == null ? null : IndexMaintainer.deserialize(localIndexBytes);
indexMaintainer = indexMaintainers.get(0);
viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan);
+ byte[] txState = scan.getAttribute(BaseScannerRegionObserver.TX_STATE);
+ tx = MutationState.decodeTransaction(txState);
}
final TupleProjector p = TupleProjector.deserializeProjectorFromScan(scan);
final HashJoinInfo j = HashJoinInfo.deserializeHashJoinFromScan(scan);
innerScanner =
getWrappedScanner(c, innerScanner, arrayKVRefs, arrayFuncRefs, offset, scan,
- dataColumns, tupleProjector, dataRegion, indexMaintainer, viewConstants,
- kvSchema, kvSchemaBitSet, j == null ? p : null, ptr);
+ dataColumns, tupleProjector, dataRegion, indexMaintainer, tx,
+ viewConstants, kvSchema, kvSchemaBitSet, j == null ? p : null, ptr);
final ImmutableBytesWritable tenantId = ScanUtil.getTenantId(scan);
if (j != null) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
index 2f31b08..9f3bdb4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
@@ -26,18 +26,17 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
import org.apache.phoenix.cache.GlobalCache;
import org.apache.phoenix.cache.TenantCache;
-import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
-
import org.apache.phoenix.coprocessor.ServerCachingProtocol.ServerCacheFactory;
import org.apache.phoenix.coprocessor.generated.ServerCachingProtos.AddServerCacheRequest;
import org.apache.phoenix.coprocessor.generated.ServerCachingProtos.AddServerCacheResponse;
import org.apache.phoenix.coprocessor.generated.ServerCachingProtos.RemoveServerCacheRequest;
import org.apache.phoenix.coprocessor.generated.ServerCachingProtos.RemoveServerCacheResponse;
import org.apache.phoenix.coprocessor.generated.ServerCachingProtos.ServerCachingService;
+import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.protobuf.ProtobufUtil;
+import org.apache.phoenix.util.ByteUtil;
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
@@ -66,6 +65,7 @@ public class ServerCachingEndpointImpl extends ServerCachingService implements C
ImmutableBytesWritable cachePtr =
org.apache.phoenix.protobuf.ProtobufUtil
.toImmutableBytesWritable(request.getCachePtr());
+ byte[] txState = request.hasTxState() ? request.getTxState().toByteArray() : ByteUtil.EMPTY_BYTE_ARRAY;
try {
@SuppressWarnings("unchecked")
@@ -73,7 +73,7 @@ public class ServerCachingEndpointImpl extends ServerCachingService implements C
(Class<ServerCacheFactory>) Class.forName(request.getCacheFactory().getClassName());
ServerCacheFactory cacheFactory = serverCacheFactoryClass.newInstance();
tenantCache.addServerCache(new ImmutableBytesPtr(request.getCacheId().toByteArray()),
- cachePtr, cacheFactory);
+ cachePtr, txState, cacheFactory);
} catch (Throwable e) {
ProtobufUtil.setControllerException(controller, new IOException(e));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
index 4fdfe99..b201c8e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
@@ -36,18 +36,19 @@ import org.apache.phoenix.memory.MemoryManager.MemoryChunk;
*/
public interface ServerCachingProtocol {
public static interface ServerCacheFactory extends Writable {
- public Closeable newCache(ImmutableBytesWritable cachePtr, MemoryChunk chunk) throws SQLException;
+ public Closeable newCache(ImmutableBytesWritable cachePtr, byte[] txState, MemoryChunk chunk) throws SQLException;
}
/**
* Add the cache to the region server cache.
* @param tenantId the tenantId or null if not applicable
* @param cacheId unique identifier of the cache
* @param cachePtr pointer to the byte array of the cache
+ * @param txState TODO
* @param cacheFactory factory that converts from byte array to object representation on the server side
* @return true on success and otherwise throws
* @throws SQLException
*/
- public boolean addServerCache(byte[] tenantId, byte[] cacheId, ImmutableBytesWritable cachePtr, ServerCacheFactory cacheFactory) throws SQLException;
+ public boolean addServerCache(byte[] tenantId, byte[] cacheId, ImmutableBytesWritable cachePtr, byte[] txState, ServerCacheFactory cacheFactory) throws SQLException;
/**
* Remove the cache from the region server cache. Called upon completion of
* the operation when cache is no longer needed.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 5133cb2..ffdc05f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -105,6 +105,8 @@ import org.apache.phoenix.util.TimeKeeper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import co.cask.tephra.TxConstants;
+
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -403,6 +405,8 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
Delete delete = new Delete(firstKV.getRowArray(),
firstKV.getRowOffset(), firstKV.getRowLength(),ts);
mutations.add(delete);
+ // force tephra to ignore this deletes
+ delete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
} else if (isUpsert) {
Arrays.fill(values, null);
int i = 0;
@@ -462,6 +466,8 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
results.get(0).getRowOffset(),
results.get(0).getRowLength());
delete.deleteColumns(deleteCF, deleteCQ, ts);
+ // force tephra to ignore this deletes
+ delete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
mutations.add(delete);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
index 25f8271..7e71cd9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
@@ -3218,6 +3218,16 @@ public final class PTableProtos {
* <code>optional bool rowKeyOrderOptimizable = 26;</code>
*/
boolean getRowKeyOrderOptimizable();
+
+ // optional bool transactional = 27;
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ boolean hasTransactional();
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ boolean getTransactional();
}
/**
* Protobuf type {@code PTable}
@@ -3418,6 +3428,11 @@ public final class PTableProtos {
rowKeyOrderOptimizable_ = input.readBool();
break;
}
+ case 216: {
+ bitField0_ |= 0x00400000;
+ transactional_ = input.readBool();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -3980,6 +3995,22 @@ public final class PTableProtos {
return rowKeyOrderOptimizable_;
}
+ // optional bool transactional = 27;
+ public static final int TRANSACTIONAL_FIELD_NUMBER = 27;
+ private boolean transactional_;
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ public boolean hasTransactional() {
+ return ((bitField0_ & 0x00400000) == 0x00400000);
+ }
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ public boolean getTransactional() {
+ return transactional_;
+ }
+
private void initFields() {
schemaNameBytes_ = com.google.protobuf.ByteString.EMPTY;
tableNameBytes_ = com.google.protobuf.ByteString.EMPTY;
@@ -4007,6 +4038,7 @@ public final class PTableProtos {
storeNulls_ = false;
baseColumnCount_ = 0;
rowKeyOrderOptimizable_ = false;
+ transactional_ = false;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -4152,6 +4184,9 @@ public final class PTableProtos {
if (((bitField0_ & 0x00200000) == 0x00200000)) {
output.writeBool(26, rowKeyOrderOptimizable_);
}
+ if (((bitField0_ & 0x00400000) == 0x00400000)) {
+ output.writeBool(27, transactional_);
+ }
getUnknownFields().writeTo(output);
}
@@ -4270,6 +4305,10 @@ public final class PTableProtos {
size += com.google.protobuf.CodedOutputStream
.computeBoolSize(26, rowKeyOrderOptimizable_);
}
+ if (((bitField0_ & 0x00400000) == 0x00400000)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBoolSize(27, transactional_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -4411,6 +4450,11 @@ public final class PTableProtos {
result = result && (getRowKeyOrderOptimizable()
== other.getRowKeyOrderOptimizable());
}
+ result = result && (hasTransactional() == other.hasTransactional());
+ if (hasTransactional()) {
+ result = result && (getTransactional()
+ == other.getTransactional());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -4528,6 +4572,10 @@ public final class PTableProtos {
hash = (37 * hash) + ROWKEYORDEROPTIMIZABLE_FIELD_NUMBER;
hash = (53 * hash) + hashBoolean(getRowKeyOrderOptimizable());
}
+ if (hasTransactional()) {
+ hash = (37 * hash) + TRANSACTIONAL_FIELD_NUMBER;
+ hash = (53 * hash) + hashBoolean(getTransactional());
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -4704,6 +4752,8 @@ public final class PTableProtos {
bitField0_ = (bitField0_ & ~0x01000000);
rowKeyOrderOptimizable_ = false;
bitField0_ = (bitField0_ & ~0x02000000);
+ transactional_ = false;
+ bitField0_ = (bitField0_ & ~0x04000000);
return this;
}
@@ -4852,6 +4902,10 @@ public final class PTableProtos {
to_bitField0_ |= 0x00200000;
}
result.rowKeyOrderOptimizable_ = rowKeyOrderOptimizable_;
+ if (((from_bitField0_ & 0x04000000) == 0x04000000)) {
+ to_bitField0_ |= 0x00400000;
+ }
+ result.transactional_ = transactional_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -5024,6 +5078,9 @@ public final class PTableProtos {
if (other.hasRowKeyOrderOptimizable()) {
setRowKeyOrderOptimizable(other.getRowKeyOrderOptimizable());
}
+ if (other.hasTransactional()) {
+ setTransactional(other.getTransactional());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -6694,6 +6751,39 @@ public final class PTableProtos {
return this;
}
+ // optional bool transactional = 27;
+ private boolean transactional_ ;
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ public boolean hasTransactional() {
+ return ((bitField0_ & 0x04000000) == 0x04000000);
+ }
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ public boolean getTransactional() {
+ return transactional_;
+ }
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ public Builder setTransactional(boolean value) {
+ bitField0_ |= 0x04000000;
+ transactional_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bool transactional = 27;</code>
+ */
+ public Builder clearTransactional() {
+ bitField0_ = (bitField0_ & ~0x04000000);
+ transactional_ = false;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:PTable)
}
@@ -6740,7 +6830,7 @@ public final class PTableProtos {
"leStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030\002 \003(\014\022\033\n\023" +
"guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyBytesCou",
"nt\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n\013pGu" +
- "idePosts\030\006 \001(\0132\014.PGuidePosts\"\357\004\n\006PTable\022" +
+ "idePosts\030\006 \001(\0132\014.PGuidePosts\"\206\005\n\006PTable\022" +
"\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tableNameByt" +
"es\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.PTableType" +
"\022\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenceNumber\030\005" +
@@ -6756,10 +6846,11 @@ public final class PTableProtos {
"wIndexId\030\025 \001(\005\022\021\n\tindexType\030\026 \001(\014\022\026\n\016sta" +
"tsTimeStamp\030\027 \001(\003\022\022\n\nstoreNulls\030\030 \001(\010\022\027\n" +
"\017baseColumnCount\030\031 \001(\005\022\036\n\026rowKeyOrderOpt" +
- "imizable\030\032 \001(\010*A\n\nPTableType\022\n\n\006SYSTEM\020\000" +
- "\022\010\n\004USER\020\001\022\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020" +
- "\004B@\n(org.apache.phoenix.coprocessor.gene",
- "ratedB\014PTableProtosH\001\210\001\001\240\001\001"
+ "imizable\030\032 \001(\010\022\025\n\rtransactional\030\033 \001(\010*A\n" +
+ "\nPTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIE" +
+ "W\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.p",
+ "hoenix.coprocessor.generatedB\014PTableProt" +
+ "osH\001\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -6783,7 +6874,7 @@ public final class PTableProtos {
internal_static_PTable_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_PTable_descriptor,
- new java.lang.String[] { "SchemaNameBytes", "TableNameBytes", "TableType", "IndexState", "SequenceNumber", "TimeStamp", "PkNameBytes", "BucketNum", "Columns", "Indexes", "IsImmutableRows", "GuidePosts", "DataTableNameBytes", "DefaultFamilyName", "DisableWAL", "MultiTenant", "ViewType", "ViewStatement", "PhysicalNames", "TenantId", "ViewIndexId", "IndexType", "StatsTimeStamp", "StoreNulls", "BaseColumnCount", "RowKeyOrderOptimizable", });
+ new java.lang.String[] { "SchemaNameBytes", "TableNameBytes", "TableType", "IndexState", "SequenceNumber", "TimeStamp", "PkNameBytes", "BucketNum", "Columns", "Indexes", "IsImmutableRows", "GuidePosts", "DataTableNameBytes", "DefaultFamilyName", "DisableWAL", "MultiTenant", "ViewType", "ViewStatement", "PhysicalNames", "TenantId", "ViewIndexId", "IndexType", "StatsTimeStamp", "StoreNulls", "BaseColumnCount", "RowKeyOrderOptimizable", "Transactional", });
return null;
}
};
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
index 69db21b..5ee1dfb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
@@ -699,6 +699,16 @@ public final class ServerCachingProtos {
* <code>required .ServerCacheFactory cacheFactory = 4;</code>
*/
org.apache.phoenix.coprocessor.generated.ServerCacheFactoryProtos.ServerCacheFactoryOrBuilder getCacheFactoryOrBuilder();
+
+ // optional bytes txState = 5;
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ boolean hasTxState();
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ com.google.protobuf.ByteString getTxState();
}
/**
* Protobuf type {@code AddServerCacheRequest}
@@ -787,6 +797,11 @@ public final class ServerCachingProtos {
bitField0_ |= 0x00000008;
break;
}
+ case 42: {
+ bitField0_ |= 0x00000010;
+ txState_ = input.readBytes();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -903,11 +918,28 @@ public final class ServerCachingProtos {
return cacheFactory_;
}
+ // optional bytes txState = 5;
+ public static final int TXSTATE_FIELD_NUMBER = 5;
+ private com.google.protobuf.ByteString txState_;
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ public boolean hasTxState() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ public com.google.protobuf.ByteString getTxState() {
+ return txState_;
+ }
+
private void initFields() {
tenantId_ = com.google.protobuf.ByteString.EMPTY;
cacheId_ = com.google.protobuf.ByteString.EMPTY;
cachePtr_ = org.apache.phoenix.coprocessor.generated.ServerCachingProtos.ImmutableBytesWritable.getDefaultInstance();
cacheFactory_ = org.apache.phoenix.coprocessor.generated.ServerCacheFactoryProtos.ServerCacheFactory.getDefaultInstance();
+ txState_ = com.google.protobuf.ByteString.EMPTY;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -953,6 +985,9 @@ public final class ServerCachingProtos {
if (((bitField0_ & 0x00000008) == 0x00000008)) {
output.writeMessage(4, cacheFactory_);
}
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ output.writeBytes(5, txState_);
+ }
getUnknownFields().writeTo(output);
}
@@ -978,6 +1013,10 @@ public final class ServerCachingProtos {
size += com.google.protobuf.CodedOutputStream
.computeMessageSize(4, cacheFactory_);
}
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(5, txState_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -1021,6 +1060,11 @@ public final class ServerCachingProtos {
result = result && getCacheFactory()
.equals(other.getCacheFactory());
}
+ result = result && (hasTxState() == other.hasTxState());
+ if (hasTxState()) {
+ result = result && getTxState()
+ .equals(other.getTxState());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -1050,6 +1094,10 @@ public final class ServerCachingProtos {
hash = (37 * hash) + CACHEFACTORY_FIELD_NUMBER;
hash = (53 * hash) + getCacheFactory().hashCode();
}
+ if (hasTxState()) {
+ hash = (37 * hash) + TXSTATE_FIELD_NUMBER;
+ hash = (53 * hash) + getTxState().hashCode();
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -1177,6 +1225,8 @@ public final class ServerCachingProtos {
cacheFactoryBuilder_.clear();
}
bitField0_ = (bitField0_ & ~0x00000008);
+ txState_ = com.google.protobuf.ByteString.EMPTY;
+ bitField0_ = (bitField0_ & ~0x00000010);
return this;
}
@@ -1229,6 +1279,10 @@ public final class ServerCachingProtos {
} else {
result.cacheFactory_ = cacheFactoryBuilder_.build();
}
+ if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+ to_bitField0_ |= 0x00000010;
+ }
+ result.txState_ = txState_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -1257,6 +1311,9 @@ public final class ServerCachingProtos {
if (other.hasCacheFactory()) {
mergeCacheFactory(other.getCacheFactory());
}
+ if (other.hasTxState()) {
+ setTxState(other.getTxState());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -1610,6 +1667,42 @@ public final class ServerCachingProtos {
return cacheFactoryBuilder_;
}
+ // optional bytes txState = 5;
+ private com.google.protobuf.ByteString txState_ = com.google.protobuf.ByteString.EMPTY;
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ public boolean hasTxState() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ public com.google.protobuf.ByteString getTxState() {
+ return txState_;
+ }
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ public Builder setTxState(com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000010;
+ txState_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bytes txState = 5;</code>
+ */
+ public Builder clearTxState() {
+ bitField0_ = (bitField0_ & ~0x00000010);
+ txState_ = getDefaultInstance().getTxState();
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:AddServerCacheRequest)
}
@@ -3383,20 +3476,21 @@ public final class ServerCachingProtos {
"\n\032ServerCachingService.proto\032\030ServerCach" +
"eFactory.proto\"K\n\026ImmutableBytesWritable" +
"\022\021\n\tbyteArray\030\001 \002(\014\022\016\n\006offset\030\002 \002(\005\022\016\n\006l" +
- "ength\030\003 \002(\005\"\220\001\n\025AddServerCacheRequest\022\020\n" +
+ "ength\030\003 \002(\005\"\241\001\n\025AddServerCacheRequest\022\020\n" +
"\010tenantId\030\001 \001(\014\022\017\n\007cacheId\030\002 \002(\014\022)\n\010cach" +
"ePtr\030\003 \002(\0132\027.ImmutableBytesWritable\022)\n\014c" +
- "acheFactory\030\004 \002(\0132\023.ServerCacheFactory\"(" +
- "\n\026AddServerCacheResponse\022\016\n\006return\030\001 \002(\010" +
- "\"=\n\030RemoveServerCacheRequest\022\020\n\010tenantId" +
- "\030\001 \001(\014\022\017\n\007cacheId\030\002 \002(\014\"+\n\031RemoveServerC",
- "acheResponse\022\016\n\006return\030\001 \002(\0102\245\001\n\024ServerC" +
- "achingService\022A\n\016addServerCache\022\026.AddSer" +
- "verCacheRequest\032\027.AddServerCacheResponse" +
- "\022J\n\021removeServerCache\022\031.RemoveServerCach" +
- "eRequest\032\032.RemoveServerCacheResponseBG\n(" +
- "org.apache.phoenix.coprocessor.generated" +
- "B\023ServerCachingProtosH\001\210\001\001\240\001\001"
+ "acheFactory\030\004 \002(\0132\023.ServerCacheFactory\022\017" +
+ "\n\007txState\030\005 \001(\014\"(\n\026AddServerCacheRespons" +
+ "e\022\016\n\006return\030\001 \002(\010\"=\n\030RemoveServerCacheRe" +
+ "quest\022\020\n\010tenantId\030\001 \001(\014\022\017\n\007cacheId\030\002 \002(\014",
+ "\"+\n\031RemoveServerCacheResponse\022\016\n\006return\030" +
+ "\001 \002(\0102\245\001\n\024ServerCachingService\022A\n\016addSer" +
+ "verCache\022\026.AddServerCacheRequest\032\027.AddSe" +
+ "rverCacheResponse\022J\n\021removeServerCache\022\031" +
+ ".RemoveServerCacheRequest\032\032.RemoveServer" +
+ "CacheResponseBG\n(org.apache.phoenix.copr" +
+ "ocessor.generatedB\023ServerCachingProtosH\001" +
+ "\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -3414,7 +3508,7 @@ public final class ServerCachingProtos {
internal_static_AddServerCacheRequest_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_AddServerCacheRequest_descriptor,
- new java.lang.String[] { "TenantId", "CacheId", "CachePtr", "CacheFactory", });
+ new java.lang.String[] { "TenantId", "CacheId", "CachePtr", "CacheFactory", "TxState", });
internal_static_AddServerCacheResponse_descriptor =
getDescriptor().getMessageTypes().get(2);
internal_static_AddServerCacheResponse_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index bb76ccb..3613c95 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -159,6 +159,12 @@ public enum SQLExceptionCode {
AGGREGATE_EXPRESSION_NOT_ALLOWED_IN_INDEX(520, "42897", "Aggregate expression not allowed in an index"),
NON_DETERMINISTIC_EXPRESSION_NOT_ALLOWED_IN_INDEX(521, "42898", "Non-deterministic expression not allowed in an index"),
STATELESS_EXPRESSION_NOT_ALLOWED_IN_INDEX(522, "42899", "Stateless expression not allowed in an index"),
+
+ /**
+ * Transaction exceptions.
+ */
+ TRANSACTION_CONFLICT_EXCEPTION(523, "42900", "Transaction aborted due to conflict with other mutations"),
+ TRANSACTION_EXCEPTION(524, "42901", "Transaction aborted due to error"),
/**
* Union All related errors
@@ -258,7 +264,18 @@ public enum SQLExceptionCode {
NO_LOCAL_INDEXES(1054, "43A11", "Local secondary indexes are not supported for HBase versions " +
MetaDataUtil.decodeHBaseVersionAsString(PhoenixDatabaseMetaData.MIN_LOCAL_SI_VERSION_DISALLOW) + " through " + MetaDataUtil.decodeHBaseVersionAsString(PhoenixDatabaseMetaData.MAX_LOCAL_SI_VERSION_DISALLOW) + " inclusive."),
UNALLOWED_LOCAL_INDEXES(1055, "43A12", "Local secondary indexes are configured to not be allowed."),
+
DESC_VARBINARY_NOT_SUPPORTED(1056, "43A13", "Descending VARBINARY columns not supported"),
+
+ DEFAULT_COLUMN_FAMILY_ON_SHARED_TABLE(1069, "43A13", "Default column family not allowed on VIEW or shared INDEX"),
+ ONLY_TABLE_MAY_BE_DECLARED_TRANSACTIONAL(1070, "44A01", "Only tables may be declared as transactional"),
+ TX_MAY_NOT_SWITCH_TO_NON_TX(1071, "44A02", "A transactional table may not be switched to non transactional"),
+ STORE_NULLS_MUST_BE_TRUE_FOR_TRANSACTIONAL(1072, "44A03", "Store nulls must be true when a table is transactional"),
+ CANNOT_START_TRANSACTION_WITH_SCN_SET(1073, "44A04", "Cannot start a transaction on a connection with SCN set"),
+ TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE(1074, "44A05", "A transactional table must define VERSION of greater than one"),
+ CANNOT_SPECIFY_SCN_FOR_TXN_TABLE(1075, "44A06", "Cannot use a connection with SCN set for a transactional table"),
+ NULL_TRANSACTION_CONTEXT(1076, "44A07", "No Tranasction Context available"),
+ TRANSACTION_FAILED(1077, "44A08", "Transaction Failure "),
/** Sequence related */
SEQUENCE_ALREADY_EXIST(1200, "42Z00", "Sequence already exists.", new Factory() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
index e292dd8..0bdb65a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
@@ -223,23 +223,26 @@ public abstract class BaseQueryPlan implements QueryPlan {
if (table.getType() != PTableType.SYSTEM) {
scan.setConsistency(connection.getConsistency());
}
- // Get the time range of row_timestamp column
- TimeRange rowTimestampRange = context.getScanRanges().getRowTimestampRange();
- // Get the already existing time range on the scan.
- TimeRange scanTimeRange = scan.getTimeRange();
- Long scn = connection.getSCN();
- if (scn == null) {
- scn = context.getCurrentTime();
- }
- try {
- TimeRange timeRangeToUse = ScanUtil.intersectTimeRange(rowTimestampRange, scanTimeRange, scn);
- if (timeRangeToUse == null) {
- return ResultIterator.EMPTY_ITERATOR;
- }
- scan.setTimeRange(timeRangeToUse.getMin(), timeRangeToUse.getMax());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ // TODO fix this in PHOENIX-2415 Support ROW_TIMESTAMP with transactional tables
+ if (!table.isTransactional()) {
+ // Get the time range of row_timestamp column
+ TimeRange rowTimestampRange = context.getScanRanges().getRowTimestampRange();
+ // Get the already existing time range on the scan.
+ TimeRange scanTimeRange = scan.getTimeRange();
+ Long scn = connection.getSCN();
+ if (scn == null) {
+ scn = context.getCurrentTime();
+ }
+ try {
+ TimeRange timeRangeToUse = ScanUtil.intersectTimeRange(rowTimestampRange, scanTimeRange, scn);
+ if (timeRangeToUse == null) {
+ return ResultIterator.EMPTY_ITERATOR;
+ }
+ scan.setTimeRange(timeRangeToUse.getMin(), timeRangeToUse.getMax());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
byte[] tenantIdBytes;
if( table.isMultiTenant() == true ) {
tenantIdBytes = connection.getTenantId() == null ? null :
@@ -326,6 +329,9 @@ public abstract class BaseQueryPlan implements QueryPlan {
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
IndexMaintainer.serialize(dataTable, ptr, indexes, context.getConnection());
scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD, ByteUtil.copyKeyBytesIfNecessary(ptr));
+ if (dataTable.isTransactional()) {
+ scan.setAttribute(BaseScannerRegionObserver.TX_STATE, context.getConnection().getMutationState().encodeTransaction());
+ }
}
private void serializeViewConstantsIntoScan(Scan scan, PTable dataTable) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java
new file mode 100644
index 0000000..6b3f9ca
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.execute;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.coprocessor.Batch.Call;
+import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
+
+import com.google.protobuf.Descriptors.MethodDescriptor;
+import com.google.protobuf.Message;
+import com.google.protobuf.Service;
+import com.google.protobuf.ServiceException;
+
+public class DelegateHTable implements HTableInterface {
+ protected final HTableInterface delegate;
+
+ public DelegateHTable(HTableInterface delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public byte[] getTableName() {
+ return delegate.getTableName();
+ }
+
+ @Override
+ public TableName getName() {
+ return delegate.getName();
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return delegate.getConfiguration();
+ }
+
+ @Override
+ public HTableDescriptor getTableDescriptor() throws IOException {
+ return delegate.getTableDescriptor();
+ }
+
+ @Override
+ public boolean exists(Get get) throws IOException {
+ return delegate.exists(get);
+ }
+
+ @Override
+ public Boolean[] exists(List<Get> gets) throws IOException {
+ return delegate.exists(gets);
+ }
+
+ @Override
+ public void batch(List<? extends Row> actions, Object[] results) throws IOException, InterruptedException {
+ delegate.batch(actions, results);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public Object[] batch(List<? extends Row> actions) throws IOException, InterruptedException {
+ return delegate.batch(actions);
+ }
+
+ @Override
+ public <R> void batchCallback(List<? extends Row> actions, Object[] results, Callback<R> callback)
+ throws IOException, InterruptedException {
+ delegate.batchCallback(actions, results, callback);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public <R> Object[] batchCallback(List<? extends Row> actions, Callback<R> callback) throws IOException,
+ InterruptedException {
+ return delegate.batchCallback(actions, callback);
+ }
+
+ @Override
+ public Result get(Get get) throws IOException {
+ return delegate.get(get);
+ }
+
+ @Override
+ public Result[] get(List<Get> gets) throws IOException {
+ return delegate.get(gets);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
+ return delegate.getRowOrBefore(row, family);
+ }
+
+ @Override
+ public ResultScanner getScanner(Scan scan) throws IOException {
+ return delegate.getScanner(scan);
+ }
+
+ @Override
+ public ResultScanner getScanner(byte[] family) throws IOException {
+ return delegate.getScanner(family);
+ }
+
+ @Override
+ public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
+ return delegate.getScanner(family, qualifier);
+ }
+
+ @Override
+ public void put(Put put) throws IOException {
+ delegate.put(put);
+ }
+
+ @Override
+ public void put(List<Put> puts) throws IOException {
+ delegate.put(puts);
+ }
+
+ @Override
+ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, byte[] value, Put put) throws IOException {
+ return delegate.checkAndPut(row, family, qualifier, value, put);
+ }
+
+ @Override
+ public void delete(Delete delete) throws IOException {
+ delegate.delete(delete);
+ }
+
+ @Override
+ public void delete(List<Delete> deletes) throws IOException {
+ delegate.delete(deletes);
+ }
+
+ @Override
+ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, byte[] value, Delete delete)
+ throws IOException {
+ return delegate.checkAndDelete(row, family, qualifier, value, delete);
+ }
+
+ @Override
+ public void mutateRow(RowMutations rm) throws IOException {
+ delegate.mutateRow(rm);
+ }
+
+ @Override
+ public Result append(Append append) throws IOException {
+ return delegate.append(append);
+ }
+
+ @Override
+ public Result increment(Increment increment) throws IOException {
+ return delegate.increment(increment);
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
+ return delegate.incrementColumnValue(row, family, qualifier, amount);
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, Durability durability)
+ throws IOException {
+ return delegate.incrementColumnValue(row, family, qualifier, amount, durability);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL)
+ throws IOException {
+ return delegate.incrementColumnValue(row, family, qualifier, amount, writeToWAL);
+ }
+
+ @Override
+ public boolean isAutoFlush() {
+ return delegate.isAutoFlush();
+ }
+
+ @Override
+ public void flushCommits() throws IOException {
+ delegate.flushCommits();
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ }
+
+ @Override
+ public CoprocessorRpcChannel coprocessorService(byte[] row) {
+ return delegate.coprocessorService(row);
+ }
+
+ @Override
+ public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
+ Call<T, R> callable) throws ServiceException, Throwable {
+ return delegate.coprocessorService(service, startKey, endKey, callable);
+ }
+
+ @Override
+ public <T extends Service, R> void coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
+ Call<T, R> callable, Callback<R> callback) throws ServiceException, Throwable {
+ delegate.coprocessorService(service, startKey, endKey, callable, callback);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void setAutoFlush(boolean autoFlush) {
+ delegate.setAutoFlush(autoFlush);
+ }
+
+ @Override
+ public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
+ delegate.setAutoFlush(autoFlush, clearBufferOnFail);
+ }
+
+ @Override
+ public void setAutoFlushTo(boolean autoFlush) {
+ delegate.setAutoFlushTo(autoFlush);
+ }
+
+ @Override
+ public long getWriteBufferSize() {
+ return delegate.getWriteBufferSize();
+ }
+
+ @Override
+ public void setWriteBufferSize(long writeBufferSize) throws IOException {
+ delegate.setWriteBufferSize(writeBufferSize);
+ }
+
+ @Override
+ public <R extends Message> Map<byte[], R> batchCoprocessorService(MethodDescriptor methodDescriptor,
+ Message request, byte[] startKey, byte[] endKey, R responsePrototype) throws ServiceException, Throwable {
+ return delegate.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype);
+ }
+
+ @Override
+ public <R extends Message> void batchCoprocessorService(MethodDescriptor methodDescriptor, Message request,
+ byte[] startKey, byte[] endKey, R responsePrototype, Callback<R> callback) throws ServiceException,
+ Throwable {
+ delegate.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype, callback);
+ }
+
+ @Override
+ public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, CompareOp compareOp, byte[] value,
+ RowMutations mutation) throws IOException {
+ return delegate.checkAndMutate(row, family, qualifier, compareOp, value, mutation);
+ }
+
+ @Override
+ public boolean[] existsAll(List<Get> gets) throws IOException {
+ return delegate.existsAll(gets);
+ }
+
+ @Override
+ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier,
+ CompareOp compareOp, byte[] value, Put put) throws IOException {
+ return delegate.checkAndPut(row, family, qualifier, value, put);
+ }
+
+ @Override
+ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier,
+ CompareOp compareOp, byte[] value, Delete delete)
+ throws IOException {
+ return delegate.checkAndDelete(row, family, qualifier, compareOp, value, delete);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
index 72920b2..cf89380 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
@@ -23,6 +23,7 @@ import static org.apache.phoenix.util.LogUtil.addCustomAnnotations;
import java.sql.SQLException;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -66,6 +67,7 @@ import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PArrayDataType;
import org.apache.phoenix.schema.types.PBoolean;
@@ -75,6 +77,7 @@ import org.apache.phoenix.util.SQLCloseable;
import org.apache.phoenix.util.SQLCloseables;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
public class HashJoinPlan extends DelegateQueryPlan {
private static final Log LOG = LogFactory.getLog(HashJoinPlan.class);
@@ -83,6 +86,7 @@ public class HashJoinPlan extends DelegateQueryPlan {
private final HashJoinInfo joinInfo;
private final SubPlan[] subPlans;
private final boolean recompileWhereClause;
+ private final Set<TableRef> tableRefs;
private final int maxServerCacheTimeToLive;
private List<SQLCloseable> dependencies;
private HashCacheClient hashClient;
@@ -114,11 +118,21 @@ public class HashJoinPlan extends DelegateQueryPlan {
this.joinInfo = joinInfo;
this.subPlans = subPlans;
this.recompileWhereClause = recompileWhereClause;
+ this.tableRefs = Sets.newHashSetWithExpectedSize(subPlans.length + plan.getSourceRefs().size());
+ this.tableRefs.addAll(plan.getSourceRefs());
+ for (SubPlan subPlan : subPlans) {
+ tableRefs.addAll(subPlan.getInnerPlan().getSourceRefs());
+ }
this.maxServerCacheTimeToLive = plan.getContext().getConnection().getQueryServices().getProps().getInt(
QueryServices.MAX_SERVER_CACHE_TIME_TO_LIVE_MS_ATTRIB, QueryServicesOptions.DEFAULT_MAX_SERVER_CACHE_TIME_TO_LIVE_MS);
}
@Override
+ public Set<TableRef> getSourceRefs() {
+ return tableRefs;
+ }
+
+ @Override
public ResultIterator iterator() throws SQLException {
return iterator(DefaultParallelScanGrouper.getInstance());
}
@@ -251,6 +265,7 @@ public class HashJoinPlan extends DelegateQueryPlan {
public void postProcess(Object result, HashJoinPlan parent) throws SQLException;
public List<String> getPreSteps(HashJoinPlan parent) throws SQLException;
public List<String> getPostSteps(HashJoinPlan parent) throws SQLException;
+ public QueryPlan getInnerPlan();
}
public static class WhereClauseSubPlan implements SubPlan {
@@ -321,6 +336,11 @@ public class HashJoinPlan extends DelegateQueryPlan {
public List<String> getPostSteps(HashJoinPlan parent) throws SQLException {
return Collections.<String>emptyList();
}
+
+ @Override
+ public QueryPlan getInnerPlan() {
+ return plan;
+ }
}
public static class HashSubPlan implements SubPlan {
@@ -411,7 +431,12 @@ public class HashJoinPlan extends DelegateQueryPlan {
+ " IN (" + keyRangeRhsExpression.toString() + ")";
return Collections.<String> singletonList(step);
}
-
+
+
+ @Override
+ public QueryPlan getInnerPlan() {
+ return plan;
+ }
}
}
[12/52] [abbrv] phoenix git commit: PHOENIX-2451 Use ReadOnlyProps
instead of Configuration in ConnectionQueryServicesImpl
Posted by ma...@apache.org.
PHOENIX-2451 Use ReadOnlyProps instead of Configuration in ConnectionQueryServicesImpl
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b621c7fd
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b621c7fd
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b621c7fd
Branch: refs/heads/calcite
Commit: b621c7fdc9ca29acfd41bb206ace679eb67ed7bd
Parents: dec2ad9
Author: James Taylor <jt...@salesforce.com>
Authored: Tue Nov 24 20:36:31 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Tue Nov 24 20:36:31 2015 -0800
----------------------------------------------------------------------
.../phoenix/query/ConnectionQueryServicesImpl.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b621c7fd/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 6d2cea1..17df42d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -295,7 +295,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
.maximumSize(MAX_TABLE_STATS_CACHE_ENTRIES)
.expireAfterWrite(halfStatsUpdateFreq, TimeUnit.MILLISECONDS)
.build();
- this.returnSequenceValues = config.getBoolean(QueryServices.RETURN_SEQUENCE_VALUES_ATTRIB, QueryServicesOptions.DEFAULT_RETURN_SEQUENCE_VALUES);
+ this.returnSequenceValues = props.getBoolean(QueryServices.RETURN_SEQUENCE_VALUES_ATTRIB, QueryServicesOptions.DEFAULT_RETURN_SEQUENCE_VALUES);
}
@Override
@@ -309,7 +309,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
ZKClients.reWatchOnExpire(
ZKClients.retryOnFailure(
ZKClientService.Builder.of(zkQuorumServersString)
- .setSessionTimeout(config.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT))
+ .setSessionTimeout(props.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT))
.build(),
RetryStrategies.exponentialDelay(500, 2000, TimeUnit.MILLISECONDS)
)
@@ -333,7 +333,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
User.login(config, HBASE_CLIENT_KEYTAB, HBASE_CLIENT_PRINCIPAL, null);
logger.info("Successfull login to secure cluster!!");
}
- boolean transactionsEnabled = config.getBoolean(
+ boolean transactionsEnabled = props.getBoolean(
QueryServices.TRANSACTIONS_ENABLED,
QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED);
// only initialize the tx service client if needed
@@ -922,7 +922,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
boolean tableExist = true;
try {
final String quorum = ZKConfig.getZKQuorumServersString(config);
- final String znode = config.get(HConstants.ZOOKEEPER_ZNODE_PARENT);
+ final String znode = this.props.get(HConstants.ZOOKEEPER_ZNODE_PARENT);
logger.debug("Found quorum: " + quorum + ":" + znode);
admin = new HBaseAdmin(config);
try {
@@ -1184,7 +1184,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private void ensureViewIndexTableCreated(byte[] physicalTableName, Map<String,Object> tableProps, List<Pair<byte[],Map<String,Object>>> families, byte[][] splits, long timestamp) throws SQLException {
Long maxFileSize = (Long)tableProps.get(HTableDescriptor.MAX_FILESIZE);
if (maxFileSize == null) {
- maxFileSize = this.config.getLong(HConstants.HREGION_MAX_FILESIZE, HConstants.DEFAULT_MAX_FILE_SIZE);
+ maxFileSize = this.props.getLong(HConstants.HREGION_MAX_FILESIZE, HConstants.DEFAULT_MAX_FILE_SIZE);
}
byte[] physicalIndexName = MetaDataUtil.getViewIndexPhysicalName(physicalTableName);
@@ -1192,7 +1192,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
// Get percentage to use from table props first and then fallback to config
Integer indexMaxFileSizePercProp = (Integer)tableProps.remove(QueryServices.INDEX_MAX_FILESIZE_PERC_ATTRIB);
if (indexMaxFileSizePercProp == null) {
- indexMaxFileSizePerc = config.getInt(QueryServices.INDEX_MAX_FILESIZE_PERC_ATTRIB, QueryServicesOptions.DEFAULT_INDEX_MAX_FILESIZE_PERC);
+ indexMaxFileSizePerc = this.props.getInt(QueryServices.INDEX_MAX_FILESIZE_PERC_ATTRIB, QueryServicesOptions.DEFAULT_INDEX_MAX_FILESIZE_PERC);
} else {
indexMaxFileSizePerc = indexMaxFileSizePercProp;
}
[29/52] [abbrv] phoenix git commit: PHOENIX-2374 Honor
PTable.isWALDisabled() for row deletes
Posted by ma...@apache.org.
PHOENIX-2374 Honor PTable.isWALDisabled() for row deletes
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/15ab697f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/15ab697f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/15ab697f
Branch: refs/heads/calcite
Commit: 15ab697fa0958424a165862de411a8253e184270
Parents: d4a4376
Author: James Taylor <jt...@salesforce.com>
Authored: Mon Nov 30 20:07:43 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Nov 30 20:07:43 2015 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/schema/PTableImpl.java | 17 +++--
.../org/apache/phoenix/schema/MutationTest.java | 74 ++++++++++++++++++++
2 files changed, 84 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15ab697f/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index 0827ea7..1805d94 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -697,12 +697,15 @@ public class PTableImpl implements PTable {
newMutations();
}
- @SuppressWarnings("deprecation")
private void newMutations() {
- this.setValues = new Put(this.key);
- this.unsetValues = new Delete(this.key);
- this.setValues.setWriteToWAL(!isWALDisabled());
- this.unsetValues.setWriteToWAL(!isWALDisabled());
+ Put put = new Put(this.key);
+ Delete delete = new Delete(this.key);
+ if (isWALDisabled()) {
+ put.setDurability(Durability.SKIP_WAL);
+ delete.setDurability(Durability.SKIP_WAL);
+ }
+ this.setValues = put;
+ this.unsetValues = delete;
}
@Override
@@ -781,6 +784,7 @@ public class PTableImpl implements PTable {
}
}
+ @SuppressWarnings("deprecation")
@Override
public void delete() {
newMutations();
@@ -800,8 +804,7 @@ public class PTableImpl implements PTable {
}
deleteRow = delete;
}
- // No need to write to the WAL for indexes
- if (PTableImpl.this.getType() == PTableType.INDEX) {
+ if (isWALDisabled()) {
deleteRow.setDurability(Durability.SKIP_WAL);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15ab697f/phoenix-core/src/test/java/org/apache/phoenix/schema/MutationTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/MutationTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/MutationTest.java
new file mode 100644
index 0000000..ccbda54
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/MutationTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.BaseConnectionlessQueryTest;
+import org.junit.Test;
+
+public class MutationTest extends BaseConnectionlessQueryTest {
+ @Test
+ public void testDurability() throws Exception {
+ testDurability(true);
+ testDurability(false);
+ }
+
+ private void testDurability(boolean disableWAL) throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ try {
+ Durability expectedDurability = disableWAL ? Durability.SKIP_WAL : Durability.USE_DEFAULT;
+ conn.setAutoCommit(false);
+ conn.createStatement().execute("CREATE TABLE t1 (k integer not null primary key, a.k varchar, b.k varchar) " + (disableWAL ? "DISABLE_WAL=true" : ""));
+ conn.createStatement().execute("UPSERT INTO t1 VALUES(1,'a','b')");
+ conn.createStatement().execute("DELETE FROM t1 WHERE k=2");
+ assertDurability(conn,expectedDurability);
+ conn.createStatement().execute("DELETE FROM t1 WHERE k=1");
+ assertDurability(conn,expectedDurability);
+ conn.createStatement().execute("DROP TABLE t1");
+ } finally {
+ conn.close();
+ }
+ }
+
+ private void assertDurability(Connection conn, Durability durability) throws SQLException {
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ Iterator<Pair<byte[], List<Mutation>>> it = pconn.getMutationState().toMutations();
+ assertTrue(it.hasNext());
+ while (it.hasNext()) {
+ Pair<byte[], List<Mutation>> pair = it.next();
+ assertFalse(pair.getSecond().isEmpty());
+ for (Mutation m : pair.getSecond()) {
+ assertEquals(durability, m.getDurability());
+ }
+ }
+ }
+
+}
[13/52] [abbrv] phoenix git commit: PHOENIX-2441 Addendum to not
require transaction manager to be run
Posted by ma...@apache.org.
PHOENIX-2441 Addendum to not require transaction manager to be run
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/104af256
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/104af256
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/104af256
Branch: refs/heads/calcite
Commit: 104af256a0e92f2bab3f99df71fdef92cc0e1159
Parents: b621c7f
Author: James Taylor <jt...@salesforce.com>
Authored: Tue Nov 24 21:25:25 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Tue Nov 24 21:25:25 2015 -0800
----------------------------------------------------------------------
.../apache/phoenix/tx/TxPointInTimeQueryIT.java | 62 --------------------
.../phoenix/compile/QueryCompilerTest.java | 22 ++++++-
2 files changed, 20 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/104af256/phoenix-core/src/it/java/org/apache/phoenix/tx/TxPointInTimeQueryIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TxPointInTimeQueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/TxPointInTimeQueryIT.java
deleted file mode 100644
index a3b3aa2..0000000
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/TxPointInTimeQueryIT.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.tx;
-
-import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.util.Properties;
-
-import org.apache.phoenix.end2end.BaseClientManagedTimeIT;
-import org.apache.phoenix.exception.SQLExceptionCode;
-import org.apache.phoenix.util.PhoenixRuntime;
-import org.apache.phoenix.util.PropertiesUtil;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TxPointInTimeQueryIT extends BaseClientManagedTimeIT {
-
- protected long ts;
-
- @Before
- public void initTable() throws Exception {
- ts = nextTimestamp();
- }
-
- @Test
- public void testQueryWithSCN() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts));
- try (Connection conn = DriverManager.getConnection(getUrl(), props);) {
- try {
- conn.createStatement().execute(
- "CREATE TABLE t (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR) TRANSACTIONAL=true");
- fail();
- } catch (SQLException e) {
- assertEquals("Unexpected Exception",
- SQLExceptionCode.CANNOT_START_TRANSACTION_WITH_SCN_SET
- .getErrorCode(), e.getErrorCode());
- }
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/104af256/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
index 9da1a67..bea222b 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
@@ -2126,7 +2126,25 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest {
} catch (SQLException e) {
assertEquals(SQLExceptionCode.ROWTIMESTAMP_COL_INVALID_TYPE.getErrorCode(), e.getErrorCode());
}
-
-
}
+
+ @Test
+ public void testQueryWithSCN() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(1000));
+ props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);) {
+ try {
+ conn.createStatement().execute(
+ "CREATE TABLE t (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR) TRANSACTIONAL=true");
+ fail();
+ } catch (SQLException e) {
+ assertEquals("Unexpected Exception",
+ SQLExceptionCode.CANNOT_START_TRANSACTION_WITH_SCN_SET
+ .getErrorCode(), e.getErrorCode());
+ }
+ }
+ }
+
+
}
[19/52] [abbrv] phoenix git commit: PHOENIX-2459 Add tephra test
dependency to pherf module
Posted by ma...@apache.org.
PHOENIX-2459 Add tephra test dependency to pherf module
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/07ede93b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/07ede93b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/07ede93b
Branch: refs/heads/calcite
Commit: 07ede93b6b3ad2b0dd90c83e722395b329ce8427
Parents: b2676cb
Author: Thomas D'Silva <td...@salesforce.com>
Authored: Sat Nov 28 21:40:25 2015 -0800
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Sat Nov 28 21:40:25 2015 -0800
----------------------------------------------------------------------
phoenix-pherf/pom.xml | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/07ede93b/phoenix-pherf/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-pherf/pom.xml b/phoenix-pherf/pom.xml
index 4304ab1..209643b 100644
--- a/phoenix-pherf/pom.xml
+++ b/phoenix-pherf/pom.xml
@@ -99,6 +99,13 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <type>test-jar</type>
+ <version>${tephra.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
[11/52] [abbrv] phoenix git commit: PHOENIX-2441 Do not require
transaction manager to be run
Posted by ma...@apache.org.
PHOENIX-2441 Do not require transaction manager to be run
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/dec2ad98
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/dec2ad98
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/dec2ad98
Branch: refs/heads/calcite
Commit: dec2ad98244e519c282f9ee49741762172b2b5e0
Parents: c98de94
Author: Thomas D'Silva <td...@salesforce.com>
Authored: Tue Nov 24 16:34:06 2015 -0800
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Tue Nov 24 18:43:29 2015 -0800
----------------------------------------------------------------------
.../apache/phoenix/end2end/AlterTableIT.java | 26 ++++++++++++++++++++
.../phoenix/end2end/index/ImmutableIndexIT.java | 15 +++++++++++
.../apache/phoenix/end2end/index/IndexIT.java | 15 +++++++++++
.../end2end/index/MutableIndexFailureIT.java | 7 +++++-
.../phoenix/end2end/index/MutableIndexIT.java | 14 +++++++++++
.../end2end/index/txn/MutableRollbackIT.java | 3 ++-
.../phoenix/end2end/index/txn/RollbackIT.java | 3 ++-
.../end2end/index/txn/TxWriteFailureIT.java | 5 ++--
.../org/apache/phoenix/tx/TransactionIT.java | 14 +++++++++++
.../org/apache/phoenix/tx/TxCheckpointIT.java | 5 ++--
.../phoenix/exception/SQLExceptionCode.java | 2 ++
.../query/ConnectionQueryServicesImpl.java | 8 +++++-
.../org/apache/phoenix/query/QueryServices.java | 3 ++-
.../phoenix/query/QueryServicesOptions.java | 2 ++
.../apache/phoenix/schema/MetaDataClient.java | 23 +++++++++++++++--
15 files changed, 134 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dec2ad98/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
index cdf7b5d..c86f8c5 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
@@ -36,6 +36,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -43,6 +44,8 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.exception.SQLExceptionCode;
@@ -52,6 +55,8 @@ import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.TableNotFoundException;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
@@ -60,6 +65,8 @@ import org.apache.phoenix.util.SchemaUtil;
import org.junit.BeforeClass;
import org.junit.Test;
+import com.google.common.collect.Lists;
+
/**
*
* A lot of tests in this class test HBase level properties. As a result,
@@ -2177,5 +2184,24 @@ public class AlterTableIT extends BaseOwnClusterHBaseManagedTimeIT {
}
}
+ @Test
+ public void testCreatingTxnTableFailsIfTxnsDisabled() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ // creating a transactional table should fail if transactions are disabled
+ try {
+ conn.createStatement().execute("CREATE TABLE DEMO1(k INTEGER PRIMARY KEY, v VARCHAR) TRANSACTIONAL=true");
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.CANNOT_CREATE_TXN_TABLE_IF_TXNS_DISABLED.getErrorCode(), e.getErrorCode());
+ }
+ // altering a table to be transactional should fail if transactions are disabled
+ conn.createStatement().execute("CREATE TABLE DEMO2(k INTEGER PRIMARY KEY, v VARCHAR)");
+ try {
+ conn.createStatement().execute("ALTER TABLE DEMO2 SET TRANSACTIONAL=true");
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.CANNOT_ALTER_TO_BE_TXN_IF_TXNS_DISABLED.getErrorCode(), e.getErrorCode());
+ }
+ }
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dec2ad98/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java
index b03037c..0d329fe 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java
@@ -29,19 +29,26 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Map;
import java.util.Properties;
import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.end2end.Shadower;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
+import com.google.common.collect.Maps;
+
@RunWith(Parameterized.class)
public class ImmutableIndexIT extends BaseHBaseManagedTimeIT {
@@ -66,6 +73,14 @@ public class ImmutableIndexIT extends BaseHBaseManagedTimeIT {
this.fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
}
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(1);
+ props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
@Parameters(name="localIndex = {0} , transactional = {1}")
public static Collection<Boolean[]> data() {
return Arrays.asList(new Boolean[][] {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dec2ad98/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
index d2c6267..003e564 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
@@ -17,27 +17,34 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.phoenix.compile.ColumnResolver;
import org.apache.phoenix.compile.FromCompiler;
import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.end2end.Shadower;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.parse.NamedTableNode;
import org.apache.phoenix.parse.TableName;
import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
+import com.google.common.collect.Maps;
+
@RunWith(Parameterized.class)
public class IndexIT extends BaseHBaseManagedTimeIT {
@@ -65,6 +72,14 @@ public class IndexIT extends BaseHBaseManagedTimeIT {
this.fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
}
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(1);
+ props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
@Parameters(name="localIndex = {0} , mutable = {1} , transactional = {2}")
public static Collection<Boolean[]> data() {
return Arrays.asList(new Boolean[][] {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dec2ad98/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index 39c90be..7c44401 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -35,6 +35,7 @@ import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.Timer;
import java.util.TimerTask;
@@ -80,6 +81,8 @@ import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
+
+import com.google.common.collect.Maps;
/**
*
* Test for failure of region server to write to index table.
@@ -131,7 +134,9 @@ public class MutableIndexFailureIT extends BaseTest {
String clientPort = util.getConfiguration().get(QueryServices.ZOOKEEPER_PORT_ATTRIB);
url = JDBC_PROTOCOL + JDBC_PROTOCOL_SEPARATOR + LOCALHOST + JDBC_PROTOCOL_SEPARATOR + clientPort
+ JDBC_PROTOCOL_TERMINATOR + PHOENIX_TEST_DRIVER_URL_PARAM;
- driver = initAndRegisterDriver(url, ReadOnlyProps.EMPTY_PROPS);
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(1);
+ props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
+ driver = initAndRegisterDriver(url, new ReadOnlyProps(props.entrySet().iterator()));
clusterInitialized = true;
setupTxManager();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dec2ad98/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
index 8f5301c..add282e 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
@@ -31,20 +31,26 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Map;
import java.util.Properties;
import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.end2end.Shadower;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
+import com.google.common.collect.Maps;
import com.google.common.primitives.Doubles;
@RunWith(Parameterized.class)
@@ -69,6 +75,14 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT {
this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
this.fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
}
+
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(1);
+ props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
@Parameters(name="localIndex = {0} , transactional = {1}")
public static Collection<Boolean[]> data() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dec2ad98/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java
index 58f6226..e675dda 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java
@@ -73,7 +73,8 @@ public class MutableRollbackIT extends BaseHBaseManagedTimeIT {
@Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
public static void doSetup() throws Exception {
Map<String,String> props = Maps.newHashMapWithExpectedSize(2);
- props.put(QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ props.put(QueryServices.DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dec2ad98/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java
index dbddcb1..3e1d5ab 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java
@@ -67,7 +67,8 @@ public class RollbackIT extends BaseHBaseManagedTimeIT {
@Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
public static void doSetup() throws Exception {
Map<String,String> props = Maps.newHashMapWithExpectedSize(2);
- props.put(QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ props.put(QueryServices.DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dec2ad98/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java
index 0e16d97..04bf1dc 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java
@@ -100,9 +100,10 @@ public class TxWriteFailureIT extends BaseTest {
+ JDBC_PROTOCOL_SEPARATOR + clientPort
+ JDBC_PROTOCOL_TERMINATOR + PHOENIX_TEST_DRIVER_URL_PARAM;
- Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(2);
// Must update config before starting server
- props.put(QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ props.put(QueryServices.DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
driver = initAndRegisterDriver(url, new ReadOnlyProps(props.entrySet().iterator()));
clusterInitialized = true;
setupTxManager();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dec2ad98/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
index 6bcf7ea..2061f45 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
@@ -33,6 +33,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -43,24 +44,29 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.end2end.Shadower;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.TestUtil;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
import co.cask.tephra.TxConstants;
import co.cask.tephra.hbase11.coprocessor.TransactionProcessor;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
public class TransactionIT extends BaseHBaseManagedTimeIT {
@@ -70,6 +76,14 @@ public class TransactionIT extends BaseHBaseManagedTimeIT {
public void setUp() throws SQLException {
ensureTableCreated(getUrl(), TRANSACTIONAL_DATA_TABLE);
}
+
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(1);
+ props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
@Test
public void testReadOwnWrites() throws Exception {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dec2ad98/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java
index 34869dd..e9ba592 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TxCheckpointIT.java
@@ -74,8 +74,9 @@ public class TxCheckpointIT extends BaseHBaseManagedTimeIT {
@BeforeClass
@Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
public static void doSetup() throws Exception {
- Map<String,String> props = Maps.newHashMapWithExpectedSize(1);
- props.put(QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(2);
+ props.put(QueryServices.DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB, Boolean.toString(true));
+ props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dec2ad98/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index 3613c95..fb8ebfd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -276,6 +276,8 @@ public enum SQLExceptionCode {
CANNOT_SPECIFY_SCN_FOR_TXN_TABLE(1075, "44A06", "Cannot use a connection with SCN set for a transactional table"),
NULL_TRANSACTION_CONTEXT(1076, "44A07", "No Tranasction Context available"),
TRANSACTION_FAILED(1077, "44A08", "Transaction Failure "),
+ CANNOT_CREATE_TXN_TABLE_IF_TXNS_DISABLED(1078, "44A09", "Cannot create a transactional table if transactions are disabled"),
+ CANNOT_ALTER_TO_BE_TXN_IF_TXNS_DISABLED(1079, "44A10", "Cannot alter table to be transactional table if transactions are disabled"),
/** Sequence related */
SEQUENCE_ALREADY_EXIST(1200, "42Z00", "Sequence already exists.", new Factory() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dec2ad98/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index df46013..6d2cea1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -333,7 +333,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
User.login(config, HBASE_CLIENT_KEYTAB, HBASE_CLIENT_PRINCIPAL, null);
logger.info("Successfull login to secure cluster!!");
}
- initTxServiceClient();
+ boolean transactionsEnabled = config.getBoolean(
+ QueryServices.TRANSACTIONS_ENABLED,
+ QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED);
+ // only initialize the tx service client if needed
+ if (transactionsEnabled) {
+ initTxServiceClient();
+ }
this.connection = HBaseFactoryProvider.getHConnectionFactory().createConnection(this.config);
} catch (IOException e) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ESTABLISH_CONNECTION)
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dec2ad98/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 5f42ff8..ca78be0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -159,7 +159,8 @@ public interface QueryServices extends SQLCloseable {
public static final String DELAY_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.delay";
public static final String DEFAULT_KEEP_DELETED_CELLS_ATTRIB = "phoenix.table.default.keep.deleted.cells";
public static final String DEFAULT_STORE_NULLS_ATTRIB = "phoenix.table.default.store.nulls";
- public static final String DEFAULT_TRANSACTIONAL_ATTRIB = "phoenix.transactions.default.enabled";
+ public static final String DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB = "phoenix.table.istransactional.default";
+ public static final String TRANSACTIONS_ENABLED = "phoenix.transactions.enabled";
public static final String GLOBAL_METRICS_ENABLED = "phoenix.query.global.metrics.enabled";
// rpc queue configs
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dec2ad98/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 8c2f895..d42c1f7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -193,6 +193,8 @@ public class QueryServicesOptions {
// We'll also need this for transactions to work correctly
public static final boolean DEFAULT_AUTO_COMMIT = false;
public static final boolean DEFAULT_TRANSACTIONAL = false;
+ public static final boolean DEFAULT_TABLE_ISTRANSACTIONAL = false;
+ public static final boolean DEFAULT_TRANSACTIONS_ENABLED = false;
public static final boolean DEFAULT_IS_GLOBAL_METRICS_ENABLED = true;
private static final String DEFAULT_CLIENT_RPC_CONTROLLER_FACTORY = ClientRpcControllerFactory.class.getName();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dec2ad98/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 29e4af2..cabfd04 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -80,7 +80,9 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.TRANSACTIONS_ENABLED;
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_DROP_METADATA;
+import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED;
import static org.apache.phoenix.schema.PTable.ViewType.MAPPED;
import static org.apache.phoenix.schema.PTableType.TABLE;
import static org.apache.phoenix.schema.PTableType.VIEW;
@@ -451,7 +453,7 @@ public class MetaDataClient {
}
boolean defaultTransactional = connection.getQueryServices().getProps().getBoolean(
- QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB,
+ QueryServices.DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB,
QueryServicesOptions.DEFAULT_TRANSACTIONAL);
// start a txn if all table are transactional by default or if we found the table in the cache and it is transactional
// TODO if system tables become transactional remove the check
@@ -1689,12 +1691,21 @@ public class MetaDataClient {
if (parent == null) {
if (transactionalProp == null) {
transactional = connection.getQueryServices().getProps().getBoolean(
- QueryServices.DEFAULT_TRANSACTIONAL_ATTRIB,
+ QueryServices.DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB,
QueryServicesOptions.DEFAULT_TRANSACTIONAL);
} else {
transactional = transactionalProp;
}
}
+ boolean transactionsEnabled = connection.getQueryServices().getProps().getBoolean(
+ QueryServices.TRANSACTIONS_ENABLED,
+ QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED);
+ // can't create a transactional table if transactions are not enabled
+ if (!transactionsEnabled && transactional) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_CREATE_TXN_TABLE_IF_TXNS_DISABLED)
+ .setSchemaName(schemaName).setTableName(tableName)
+ .build().buildException();
+ }
tableProps.put(PhoenixDatabaseMetaData.TRANSACTIONAL, transactional);
if (transactional) {
// If TTL set, use Tephra TTL property name instead
@@ -2628,6 +2639,14 @@ public class MetaDataClient {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.TX_MAY_NOT_SWITCH_TO_NON_TX)
.setSchemaName(schemaName).setTableName(tableName).build().buildException();
}
+ // cannot create a transactional table if transactions are disabled
+ boolean transactionsEnabled = connection.getQueryServices().getProps().getBoolean(
+ QueryServices.TRANSACTIONS_ENABLED,
+ QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED);
+ if (!transactionsEnabled) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_TO_BE_TXN_IF_TXNS_DISABLED)
+ .setSchemaName(schemaName).setTableName(tableName).build().buildException();
+ }
timeStamp = TransactionUtil.getTableTimestamp(connection, isTransactional);
changingPhoenixTableProperty = true;
nonTxToTx = true;
[47/52] [abbrv] phoenix git commit: PHOENIX-2496 Attempt to
gracefully close in ShutdownHook, but still exit promptly.
Posted by ma...@apache.org.
PHOENIX-2496 Attempt to gracefully close in ShutdownHook, but still exit promptly.
The JVM will not exit until all ShutdownHooks have completed.
Blocking until a query thread completes might be on the order
of minutes which is undesirable. Try to clean up, but exit quickly.
Introduces configuration property 'phoenix.shutdown.timeoutMs'
with a default value of 5000 (5 seconds).
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b74215f8
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b74215f8
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b74215f8
Branch: refs/heads/calcite
Commit: b74215f822e38dfb7eb2d40bed1a6c82be8ae2f2
Parents: 578979a
Author: Josh Elser <el...@apache.org>
Authored: Tue Dec 8 10:59:38 2015 -0500
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Tue Dec 8 14:10:41 2015 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/jdbc/PhoenixDriver.java | 43 +++++++++++++++++++-
.../org/apache/phoenix/query/QueryServices.java | 1 +
.../phoenix/query/QueryServicesOptions.java | 1 +
3 files changed, 44 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b74215f8/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
index bb0b1e2..0f5d4aa 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDriver.java
@@ -23,16 +23,26 @@ import java.sql.SQLException;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import javax.annotation.concurrent.GuardedBy;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.ConnectionQueryServicesImpl;
import org.apache.phoenix.query.ConnectionlessQueryServicesImpl;
+import org.apache.phoenix.query.HBaseFactoryProvider;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesImpl;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,7 +74,38 @@ public final class PhoenixDriver extends PhoenixEmbeddedDriver {
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
- closeInstance(INSTANCE);
+ final Configuration config =
+ HBaseFactoryProvider.getConfigurationFactory().getConfiguration();
+ final ExecutorService svc = Executors.newSingleThreadExecutor();
+ Future<?> future = svc.submit(new Runnable() {
+ @Override
+ public void run() {
+ closeInstance(INSTANCE);
+ }
+ });
+
+ // Pull the timeout value (default 5s).
+ long millisBeforeShutdown = config.getLong(
+ QueryServices.DRIVER_SHUTDOWN_TIMEOUT_MS,
+ QueryServicesOptions.DEFAULT_DRIVER_SHUTDOWN_TIMEOUT_MS);
+
+ // Close with a timeout. If this is running, we know the JVM wants to
+ // go down. There may be other threads running that are holding the lock.
+ // We don't want to be blocked on them (for the normal HBase retry policy).
+ //
+ // We don't care about any exceptions, we're going down anyways.
+ try {
+ future.get(millisBeforeShutdown, TimeUnit.MILLISECONDS);
+ } catch (ExecutionException e) {
+ logger.warn("Failed to close instance", e);
+ } catch (InterruptedException e) {
+ logger.warn("Interrupted waiting to close instance", e);
+ } catch (TimeoutException e) {
+ logger.warn("Timed out waiting to close instance", e);
+ }
+
+ // We're going down, but try to clean up.
+ svc.shutdown();
}
});
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b74215f8/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index ca78be0..1b05334 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -52,6 +52,7 @@ public interface QueryServices extends SQLCloseable {
public static final String CONSISTENCY_ATTRIB = "phoenix.connection.consistency";
// joni byte regex engine setting
public static final String USE_BYTE_BASED_REGEX_ATTRIB = "phoenix.regex.byteBased";
+ public static final String DRIVER_SHUTDOWN_TIMEOUT_MS = "phoenix.shutdown.timeoutMs";
/**
* max size to spool the the result into
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b74215f8/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index d42c1f7..13a7beb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -109,6 +109,7 @@ public class QueryServicesOptions {
public static final boolean DEFAULT_USE_INDEXES = true; // Use indexes
public static final boolean DEFAULT_IMMUTABLE_ROWS = false; // Tables rows may be updated
public static final boolean DEFAULT_DROP_METADATA = true; // Drop meta data also.
+ public static final long DEFAULT_DRIVER_SHUTDOWN_TIMEOUT_MS = 5 * 1000; // Time to wait in ShutdownHook to exit gracefully.
public final static int DEFAULT_MUTATE_BATCH_SIZE = 1000; // Batch size for UPSERT SELECT and DELETE
// The only downside of it being out-of-sync is that the parallelization of the scan won't be as balanced as it could be.
[44/52] [abbrv] phoenix git commit: PHOENIX-2481 JSON bulkload tool
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/FormatToKeyValueMapperTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/FormatToKeyValueMapperTest.java b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/FormatToKeyValueMapperTest.java
new file mode 100644
index 0000000..3455616
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/FormatToKeyValueMapperTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce;
+
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.schema.types.PIntegerArray;
+import org.apache.phoenix.schema.types.PUnsignedInt;
+import org.apache.phoenix.util.ColumnInfo;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+import static org.junit.Assert.assertEquals;
+
+public class FormatToKeyValueMapperTest {
+
+ @Test
+ public void testBuildColumnInfoList() {
+ List<ColumnInfo> columnInfoList = ImmutableList.of(
+ new ColumnInfo("idCol", PInteger.INSTANCE.getSqlType()),
+ new ColumnInfo("unsignedIntCol", PUnsignedInt.INSTANCE.getSqlType()),
+ new ColumnInfo("stringArrayCol", PIntegerArray.INSTANCE.getSqlType()));
+
+ Configuration conf = new Configuration();
+ FormatToKeyValueMapper.configureColumnInfoList(conf, columnInfoList);
+ List<ColumnInfo> fromConfig = FormatToKeyValueMapper.buildColumnInfoList(conf);
+
+ assertEquals(columnInfoList, fromConfig);
+ }
+
+ @Test
+ public void testBuildColumnInfoList_ContainingNulls() {
+ // A null value in the column info list means "skip that column in the input"
+ List<ColumnInfo> columnInfoListWithNull = Lists.newArrayList(
+ new ColumnInfo("idCol", PInteger.INSTANCE.getSqlType()),
+ null,
+ new ColumnInfo("unsignedIntCol", PUnsignedInt.INSTANCE.getSqlType()),
+ new ColumnInfo("stringArrayCol", PIntegerArray.INSTANCE.getSqlType()));
+
+ Configuration conf = new Configuration();
+ FormatToKeyValueMapper.configureColumnInfoList(conf, columnInfoListWithNull);
+ List<ColumnInfo> fromConfig = FormatToKeyValueMapper.buildColumnInfoList(conf);
+
+ assertEquals(columnInfoListWithNull, fromConfig);
+ }
+
+ @Test
+ public void testLoadPreUpdateProcessor() {
+ Configuration conf = new Configuration();
+ conf.setClass(PhoenixConfigurationUtil.UPSERT_HOOK_CLASS_CONFKEY, MockUpsertProcessor.class,
+ ImportPreUpsertKeyValueProcessor.class);
+
+ ImportPreUpsertKeyValueProcessor processor = PhoenixConfigurationUtil.loadPreUpsertProcessor(conf);
+ assertEquals(MockUpsertProcessor.class, processor.getClass());
+ }
+
+ @Test
+ public void testLoadPreUpdateProcessor_NotConfigured() {
+
+ Configuration conf = new Configuration();
+ ImportPreUpsertKeyValueProcessor processor = PhoenixConfigurationUtil.loadPreUpsertProcessor(conf);
+
+ assertEquals(FormatToKeyValueMapper.DefaultImportPreUpsertKeyValueProcessor.class,
+ processor.getClass());
+ }
+
+ @Test(expected=IllegalStateException.class)
+ public void testLoadPreUpdateProcessor_ClassNotFound() {
+ Configuration conf = new Configuration();
+ conf.set(PhoenixConfigurationUtil.UPSERT_HOOK_CLASS_CONFKEY, "MyUndefinedClass");
+
+ PhoenixConfigurationUtil.loadPreUpsertProcessor(conf);
+ }
+
+ static class MockUpsertProcessor implements ImportPreUpsertKeyValueProcessor {
+ @Override
+ public List<KeyValue> preUpsert(byte[] rowKey, List<KeyValue> keyValues) {
+ throw new UnsupportedOperationException("Not yet implemented");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/test/java/org/apache/phoenix/util/AbstractUpsertExecutorTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/AbstractUpsertExecutorTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/AbstractUpsertExecutorTest.java
new file mode 100644
index 0000000..b614312
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/AbstractUpsertExecutorTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.phoenix.query.BaseConnectionlessQueryTest;
+import org.apache.phoenix.schema.types.PArrayDataType;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.schema.types.PIntegerArray;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableList;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+public abstract class AbstractUpsertExecutorTest<R, F> extends BaseConnectionlessQueryTest {
+
+ protected Connection conn;
+ protected List<ColumnInfo> columnInfoList;
+ protected PreparedStatement preparedStatement;
+ protected UpsertExecutor.UpsertListener<R> upsertListener;
+
+ protected abstract UpsertExecutor<R, F> getUpsertExecutor();
+ protected abstract R createRecord(Object... columnValues) throws IOException;
+
+ @Before
+ public void setUp() throws SQLException {
+ columnInfoList = ImmutableList.of(
+ new ColumnInfo("ID", Types.BIGINT),
+ new ColumnInfo("NAME", Types.VARCHAR),
+ new ColumnInfo("AGE", Types.INTEGER),
+ new ColumnInfo("VALUES", PIntegerArray.INSTANCE.getSqlType()));
+
+ preparedStatement = mock(PreparedStatement.class);
+ upsertListener = mock(UpsertExecutor.UpsertListener.class);
+ conn = DriverManager.getConnection(getUrl());
+ }
+
+ @After
+ public void tearDown() throws SQLException {
+ conn.close();
+ }
+
+ @Test
+ public void testExecute() throws Exception {
+ getUpsertExecutor().execute(createRecord(123L, "NameValue", 42, Arrays.asList(1, 2, 3)));
+
+ verify(upsertListener).upsertDone(1L);
+ verifyNoMoreInteractions(upsertListener);
+
+ verify(preparedStatement).setObject(1, Long.valueOf(123L));
+ verify(preparedStatement).setObject(2, "NameValue");
+ verify(preparedStatement).setObject(3, Integer.valueOf(42));
+ verify(preparedStatement).setObject(4, PArrayDataType.instantiatePhoenixArray(PInteger.INSTANCE, new Object[]{1,2,3}));
+ verify(preparedStatement).execute();
+ verifyNoMoreInteractions(preparedStatement);
+ }
+
+ @Test
+ public void testExecute_TooFewFields() throws Exception {
+ R recordWithTooFewFields = createRecord(123L, "NameValue");
+ getUpsertExecutor().execute(recordWithTooFewFields);
+
+ verify(upsertListener).errorOnRecord(eq(recordWithTooFewFields), any(Throwable.class));
+ verifyNoMoreInteractions(upsertListener);
+ }
+
+ @Test
+ public void testExecute_TooManyFields() throws Exception {
+ R recordWithTooManyFields = createRecord(123L, "NameValue", 42, Arrays.asList(1, 2, 3), "Garbage");
+ getUpsertExecutor().execute(recordWithTooManyFields);
+
+ verify(upsertListener).upsertDone(1L);
+ verifyNoMoreInteractions(upsertListener);
+
+ verify(preparedStatement).setObject(1, Long.valueOf(123L));
+ verify(preparedStatement).setObject(2, "NameValue");
+ verify(preparedStatement).setObject(3, Integer.valueOf(42));
+ verify(preparedStatement).setObject(4, PArrayDataType.instantiatePhoenixArray(PInteger.INSTANCE, new Object[]{1,2,3}));
+ verify(preparedStatement).execute();
+ verifyNoMoreInteractions(preparedStatement);
+ }
+
+ @Test
+ public void testExecute_NullField() throws Exception {
+ getUpsertExecutor().execute(createRecord(123L, "NameValue", null, Arrays.asList(1, 2, 3)));
+
+ verify(upsertListener).upsertDone(1L);
+ verifyNoMoreInteractions(upsertListener);
+
+ verify(preparedStatement).setObject(1, Long.valueOf(123L));
+ verify(preparedStatement).setObject(2, "NameValue");
+ verify(preparedStatement).setNull(3, columnInfoList.get(2).getSqlType());
+ verify(preparedStatement).setObject(4, PArrayDataType.instantiatePhoenixArray(PInteger.INSTANCE, new Object[]{1,2,3}));
+ verify(preparedStatement).execute();
+ verifyNoMoreInteractions(preparedStatement);
+ }
+
+ @Test
+ public void testExecute_InvalidType() throws Exception {
+ R recordWithInvalidType = createRecord(123L, "NameValue", "ThisIsNotANumber", Arrays.asList(1, 2, 3));
+ getUpsertExecutor().execute(recordWithInvalidType);
+
+ verify(upsertListener).errorOnRecord(eq(recordWithInvalidType), any(Throwable.class));
+ verifyNoMoreInteractions(upsertListener);
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/test/java/org/apache/phoenix/util/csv/CsvUpsertExecutorTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/csv/CsvUpsertExecutorTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/csv/CsvUpsertExecutorTest.java
index 6efe246..7a09bee 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/csv/CsvUpsertExecutorTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/csv/CsvUpsertExecutorTest.java
@@ -17,130 +17,50 @@
*/
package org.apache.phoenix.util.csv;
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import org.apache.commons.csv.CSVFormat;
-import org.apache.commons.csv.CSVParser;
-import org.apache.commons.csv.CSVRecord;
-import org.apache.phoenix.query.BaseConnectionlessQueryTest;
-import org.apache.phoenix.schema.types.PInteger;
-import org.apache.phoenix.schema.types.PIntegerArray;
-import org.apache.phoenix.schema.types.PArrayDataType;
-import org.apache.phoenix.util.ColumnInfo;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
import java.sql.SQLException;
-import java.sql.Types;
import java.util.List;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-
-public class CsvUpsertExecutorTest extends BaseConnectionlessQueryTest {
-
- private Connection conn;
- private List<ColumnInfo> columnInfoList;
- private PreparedStatement preparedStatement;
- private CsvUpsertExecutor.UpsertListener upsertListener;
-
- private CsvUpsertExecutor upsertExecutor;
-
- @Before
- public void setUp() throws SQLException {
- columnInfoList = ImmutableList.of(
- new ColumnInfo("ID", Types.BIGINT),
- new ColumnInfo("NAME", Types.VARCHAR),
- new ColumnInfo("AGE", Types.INTEGER),
- new ColumnInfo("VALUES", PIntegerArray.INSTANCE.getSqlType()));
-
- preparedStatement = mock(PreparedStatement.class);
- upsertListener = mock(CsvUpsertExecutor.UpsertListener.class);
- conn = DriverManager.getConnection(getUrl());
- upsertExecutor = new CsvUpsertExecutor(conn, columnInfoList, preparedStatement, upsertListener, ":");
- }
-
- @After
- public void tearDown() throws SQLException {
- conn.close();
- }
-
- @Test
- public void testExecute() throws Exception {
- upsertExecutor.execute(createCsvRecord("123,NameValue,42,1:2:3"));
-
- verify(upsertListener).upsertDone(1L);
- verifyNoMoreInteractions(upsertListener);
-
- verify(preparedStatement).setObject(1, Long.valueOf(123L));
- verify(preparedStatement).setObject(2, "NameValue");
- verify(preparedStatement).setObject(3, Integer.valueOf(42));
- verify(preparedStatement).setObject(4, PArrayDataType.instantiatePhoenixArray(PInteger.INSTANCE, new Object[]{1,2,3}));
- verify(preparedStatement).execute();
- verifyNoMoreInteractions(preparedStatement);
- }
-
- @Test
- public void testExecute_TooFewFields() throws Exception {
- CSVRecord csvRecordWithTooFewFields = createCsvRecord("123,NameValue");
- upsertExecutor.execute(csvRecordWithTooFewFields);
-
- verify(upsertListener).errorOnRecord(eq(csvRecordWithTooFewFields), any(Throwable.class));
- verifyNoMoreInteractions(upsertListener);
- }
-
- @Test
- public void testExecute_TooManyFields() throws Exception {
- CSVRecord csvRecordWithTooManyFields = createCsvRecord("123,NameValue,42,1:2:3,Garbage");
- upsertExecutor.execute(csvRecordWithTooManyFields);
-
- verify(upsertListener).upsertDone(1L);
- verifyNoMoreInteractions(upsertListener);
-
- verify(preparedStatement).setObject(1, Long.valueOf(123L));
- verify(preparedStatement).setObject(2, "NameValue");
- verify(preparedStatement).setObject(3, Integer.valueOf(42));
- verify(preparedStatement).setObject(4, PArrayDataType.instantiatePhoenixArray(PInteger.INSTANCE, new Object[]{1,2,3}));
- verify(preparedStatement).execute();
- verifyNoMoreInteractions(preparedStatement);
- }
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import org.apache.phoenix.util.AbstractUpsertExecutorTest;
+import org.apache.phoenix.util.UpsertExecutor;
+import org.junit.Before;
- @Test
- public void testExecute_NullField() throws Exception {
- upsertExecutor.execute(createCsvRecord("123,NameValue,,1:2:3"));
+import com.google.common.base.Joiner;
+import com.google.common.collect.Iterables;
- verify(upsertListener).upsertDone(1L);
- verifyNoMoreInteractions(upsertListener);
+public class CsvUpsertExecutorTest extends AbstractUpsertExecutorTest<CSVRecord, String> {
- verify(preparedStatement).setObject(1, Long.valueOf(123L));
- verify(preparedStatement).setObject(2, "NameValue");
- verify(preparedStatement).setNull(3, columnInfoList.get(2).getSqlType());
- verify(preparedStatement).setObject(4, PArrayDataType.instantiatePhoenixArray(PInteger.INSTANCE, new Object[]{1,2,3}));
- verify(preparedStatement).execute();
- verifyNoMoreInteractions(preparedStatement);
- }
+ private static final String ARRAY_SEP = ":";
- @Test
- public void testExecute_InvalidType() throws Exception {
- CSVRecord csvRecordWithInvalidType = createCsvRecord("123,NameValue,ThisIsNotANumber,1:2:3");
- upsertExecutor.execute(csvRecordWithInvalidType);
+ private UpsertExecutor<CSVRecord, String> upsertExecutor;
- verify(upsertListener).errorOnRecord(eq(csvRecordWithInvalidType), any(Throwable.class));
- verifyNoMoreInteractions(upsertListener);
+ @Override
+ public UpsertExecutor<CSVRecord, String> getUpsertExecutor() {
+ return upsertExecutor;
}
- private CSVRecord createCsvRecord(String...columnValues) throws IOException {
+ @Override
+ public CSVRecord createRecord(Object... columnValues) throws IOException {
+ for (int i = 0; i < columnValues.length; i++) {
+ if (columnValues[i] == null) {
+ // Joiner.join throws on nulls, replace with empty string.
+ columnValues[i] = "";
+ }
+ if (columnValues[i] instanceof List) {
+ columnValues[i] = Joiner.on(ARRAY_SEP).join((List<?>) columnValues[i]);
+ }
+ }
String inputRecord = Joiner.on(',').join(columnValues);
return Iterables.getFirst(CSVParser.parse(inputRecord, CSVFormat.DEFAULT), null);
}
+
+ @Before
+ public void setUp() throws SQLException {
+ super.setUp();
+ upsertExecutor = new CsvUpsertExecutor(conn, columnInfoList, preparedStatement,
+ upsertListener, ARRAY_SEP);
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/578979a1/phoenix-core/src/test/java/org/apache/phoenix/util/json/JsonUpsertExecutorTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/json/JsonUpsertExecutorTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/json/JsonUpsertExecutorTest.java
new file mode 100644
index 0000000..c042dd4
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/json/JsonUpsertExecutorTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util.json;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.phoenix.util.AbstractUpsertExecutorTest;
+import org.apache.phoenix.util.UpsertExecutor;
+import org.junit.Before;
+
+public class JsonUpsertExecutorTest extends AbstractUpsertExecutorTest<Map<?, ?>, Object> {
+
+ private UpsertExecutor<Map<?, ?>, Object> upsertExecutor;
+
+ @Override
+ protected UpsertExecutor<Map<?, ?>, Object> getUpsertExecutor() {
+ return upsertExecutor;
+ }
+
+ @Override
+ protected Map<?, ?> createRecord(Object... columnValues) throws IOException {
+ Map ret = new HashMap(columnValues.length);
+ int min = Math.min(columnInfoList.size(), columnValues.length);
+ for (int i = 0; i < min; i++) {
+ ret.put(columnInfoList.get(i).getColumnName().replace("\"", "").toLowerCase(), columnValues[i]);
+ }
+ return ret;
+ }
+
+ @Before
+ public void setUp() throws SQLException {
+ super.setUp();
+ upsertExecutor = new JsonUpsertExecutor(conn, columnInfoList, preparedStatement, upsertListener);
+ }
+}
[22/52] [abbrv] phoenix git commit: PHOENIX-2462 Add tephra test
dependency to spark module
Posted by ma...@apache.org.
PHOENIX-2462 Add tephra test dependency to spark module
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/0e45e245
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/0e45e245
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/0e45e245
Branch: refs/heads/calcite
Commit: 0e45e24581e592a21745cf07b8c7620bbd1f3263
Parents: f237982
Author: James Taylor <jt...@salesforce.com>
Authored: Sun Nov 29 16:09:53 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Sun Nov 29 16:09:53 2015 -0800
----------------------------------------------------------------------
phoenix-spark/pom.xml | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/0e45e245/phoenix-spark/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-spark/pom.xml b/phoenix-spark/pom.xml
index 5edd5be..c0ad626 100644
--- a/phoenix-spark/pom.xml
+++ b/phoenix-spark/pom.xml
@@ -478,6 +478,13 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <type>test-jar</type>
+ <version>${tephra.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
[03/52] [abbrv] phoenix git commit: PHOENIX-1674 Snapshot isolation
transaction support through Tephra (James Taylor, Thomas D'Silva)
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 18a97bd..2a08f45 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -27,6 +27,7 @@ import static org.apache.phoenix.util.UpgradeUtil.upgradeTo4_5_0;
import java.io.IOException;
import java.sql.SQLException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -115,6 +116,7 @@ import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.hbase.index.util.VersionUtil;
import org.apache.phoenix.index.PhoenixIndexBuilder;
import org.apache.phoenix.index.PhoenixIndexCodec;
+import org.apache.phoenix.index.PhoenixTransactionalIndexer;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
@@ -163,9 +165,20 @@ import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.ServerUtil;
import org.apache.phoenix.util.UpgradeUtil;
+import org.apache.twill.discovery.ZKDiscoveryService;
+import org.apache.twill.zookeeper.RetryStrategies;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.apache.twill.zookeeper.ZKClientServices;
+import org.apache.twill.zookeeper.ZKClients;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import co.cask.tephra.TransactionSystemClient;
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.distributed.PooledClientProvider;
+import co.cask.tephra.distributed.TransactionServiceClient;
+import co.cask.tephra.hbase11.coprocessor.TransactionProcessor;
+
import com.google.common.base.Joiner;
import com.google.common.base.Throwables;
import com.google.common.cache.Cache;
@@ -183,6 +196,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
// Max number of cached table stats for view or shared index physical tables
private static final int MAX_TABLE_STATS_CACHE_ENTRIES = 512;
protected final Configuration config;
+ private final ConnectionInfo connectionInfo;
// Copy of config.getProps(), but read-only to prevent synchronization that we
// don't need.
private final ReadOnlyProps props;
@@ -197,7 +211,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
// Lowest HBase version on the cluster.
private int lowestClusterHBaseVersion = Integer.MAX_VALUE;
- private boolean hasInvalidIndexConfiguration = false;
+ private boolean isMutableIndexWALCodecInstalled = true;
@GuardedBy("connectionCountLock")
private int connectionCount = 0;
@@ -205,6 +219,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private final boolean returnSequenceValues ;
private HConnection connection;
+ private TransactionServiceClient txServiceClient;
private volatile boolean initialized;
private volatile int nSequenceSaltBuckets;
@@ -257,6 +272,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
for (Entry<String,String> entry : connectionInfo.asProps()) {
config.set(entry.getKey(), entry.getValue());
}
+ this.connectionInfo = connectionInfo;
// Without making a copy of the configuration we cons up, we lose some of our properties
// on the server side during testing.
@@ -282,6 +298,30 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
this.returnSequenceValues = config.getBoolean(QueryServices.RETURN_SEQUENCE_VALUES_ATTRIB, QueryServicesOptions.DEFAULT_RETURN_SEQUENCE_VALUES);
}
+ @Override
+ public TransactionSystemClient getTransactionSystemClient() {
+ return txServiceClient;
+ }
+
+ private void initTxServiceClient() {
+ String zkQuorumServersString = connectionInfo.getZookeeperQuorum()+":"+connectionInfo.getPort();
+ ZKClientService zkClientService = ZKClientServices.delegate(
+ ZKClients.reWatchOnExpire(
+ ZKClients.retryOnFailure(
+ ZKClientService.Builder.of(zkQuorumServersString)
+ .setSessionTimeout(config.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT))
+ .build(),
+ RetryStrategies.exponentialDelay(500, 2000, TimeUnit.MILLISECONDS)
+ )
+ )
+ );
+ zkClientService.startAndWait();
+ ZKDiscoveryService zkDiscoveryService = new ZKDiscoveryService(zkClientService);
+ PooledClientProvider pooledClientProvider = new PooledClientProvider(
+ config, zkDiscoveryService);
+ this.txServiceClient = new TransactionServiceClient(config,pooledClientProvider);
+ }
+
private void openConnection() throws SQLException {
try {
// check if we need to authenticate with kerberos
@@ -293,6 +333,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
User.login(config, HBASE_CLIENT_KEYTAB, HBASE_CLIENT_PRINCIPAL, null);
logger.info("Successfull login to secure cluster!!");
}
+ initTxServiceClient();
this.connection = HBaseFactoryProvider.getHConnectionFactory().createConnection(this.config);
} catch (IOException e) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ESTABLISH_CONNECTION)
@@ -308,9 +349,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
try {
return HBaseFactoryProvider.getHTableFactory().getTable(tableName, connection, null);
} catch (org.apache.hadoop.hbase.TableNotFoundException e) {
- byte[][] schemaAndTableName = new byte[2][];
- SchemaUtil.getVarChars(tableName, schemaAndTableName);
- throw new TableNotFoundException(Bytes.toString(schemaAndTableName[0]), Bytes.toString(schemaAndTableName[1]));
+ throw new TableNotFoundException(SchemaUtil.getSchemaNameFromFullName(tableName), SchemaUtil.getTableNameFromFullName(tableName));
} catch (IOException e) {
throw new SQLException(e);
}
@@ -457,18 +496,27 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
@Override
- public PMetaData addTable(PTable table) throws SQLException {
+ public PMetaData addTable(PTable table, long resolvedTime) throws SQLException {
synchronized (latestMetaDataLock) {
try {
throwConnectionClosedIfNullMetaData();
// If existing table isn't older than new table, don't replace
// If a client opens a connection at an earlier timestamp, this can happen
- PTable existingTable = latestMetaData.getTable(new PTableKey(table.getTenantId(), table.getName().getString()));
+ PTable existingTable = latestMetaData.getTableRef(new PTableKey(table.getTenantId(), table.getName().getString())).getTable();
if (existingTable.getTimeStamp() >= table.getTimeStamp()) {
return latestMetaData;
}
} catch (TableNotFoundException e) {}
- latestMetaData = latestMetaData.addTable(table);
+ latestMetaData = latestMetaData.addTable(table, resolvedTime);
+ latestMetaDataLock.notifyAll();
+ return latestMetaData;
+ }
+ }
+
+ public PMetaData updateResolvedTimestamp(PTable table, long resolvedTime) throws SQLException {
+ synchronized (latestMetaDataLock) {
+ throwConnectionClosedIfNullMetaData();
+ latestMetaData = latestMetaData.updateResolvedTimestamp(table, resolvedTime);
latestMetaDataLock.notifyAll();
return latestMetaData;
}
@@ -490,7 +538,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
while (true) {
try {
try {
- table = metaData.getTable(new PTableKey(tenantId, tableName));
+ table = metaData.getTableRef(new PTableKey(tenantId, tableName)).getTable();
/* If the table is at the prior sequence number, then we're good to go.
* We know if we've got this far, that the server validated the mutations,
* so we'd just need to wait until the other connection that mutated the same
@@ -531,12 +579,12 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
@Override
- public PMetaData addColumn(final PName tenantId, final String tableName, final List<PColumn> columns, final long tableTimeStamp, final long tableSeqNum, final boolean isImmutableRows, final boolean isWalDisabled, final boolean isMultitenant, final boolean storeNulls) throws SQLException {
+ public PMetaData addColumn(final PName tenantId, final String tableName, final List<PColumn> columns, final long tableTimeStamp, final long tableSeqNum, final boolean isImmutableRows, final boolean isWalDisabled, final boolean isMultitenant, final boolean storeNulls, final boolean isTransactional, final long resolvedTime) throws SQLException {
return metaDataMutated(tenantId, tableName, tableSeqNum, new Mutator() {
@Override
public PMetaData mutate(PMetaData metaData) throws SQLException {
try {
- return metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls);
+ return metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime);
} catch (TableNotFoundException e) {
// The DROP TABLE may have been processed first, so just ignore.
return metaData;
@@ -556,12 +604,12 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
@Override
- public PMetaData removeColumn(final PName tenantId, final String tableName, final List<PColumn> columnsToRemove, final long tableTimeStamp, final long tableSeqNum) throws SQLException {
+ public PMetaData removeColumn(final PName tenantId, final String tableName, final List<PColumn> columnsToRemove, final long tableTimeStamp, final long tableSeqNum, final long resolvedTime) throws SQLException {
return metaDataMutated(tenantId, tableName, tableSeqNum, new Mutator() {
@Override
public PMetaData mutate(PMetaData metaData) throws SQLException {
try {
- return metaData.removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum);
+ return metaData.removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum, resolvedTime);
} catch (TableNotFoundException e) {
// The DROP TABLE may have been processed first, so just ignore.
return metaData;
@@ -594,25 +642,40 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
for (Entry<String,Object> entry : family.getSecond().entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
- columnDesc.setValue(key, value == null ? null : value.toString());
+ setHColumnDescriptorValue(columnDesc, key, value);
}
}
return columnDesc;
}
-
- private void modifyColumnFamilyDescriptor(HColumnDescriptor hcd, Pair<byte[], Map<String,Object>> family) throws SQLException {
- if (Bytes.equals(hcd.getName(), family.getFirst())) {
- modifyColumnFamilyDescriptor(hcd, family.getSecond());
+
+ // Workaround HBASE-14737
+ private static void setHColumnDescriptorValue(HColumnDescriptor columnDesc, String key, Object value) {
+ if (HConstants.VERSIONS.equals(key)) {
+ columnDesc.setMaxVersions(getMaxVersion(value));
} else {
- throw new IllegalArgumentException("Column family names don't match. Column descriptor family name: " + hcd.getNameAsString() + ", Family name: " + Bytes.toString(family.getFirst()));
+ columnDesc.setValue(key, value == null ? null : value.toString());
}
}
+
+ private static int getMaxVersion(Object value) {
+ if (value == null) {
+ return -1; // HColumnDescriptor.UNINITIALIZED is private
+ }
+ if (value instanceof Number) {
+ return ((Number)value).intValue();
+ }
+ String stringValue = value.toString();
+ if (stringValue.isEmpty()) {
+ return -1;
+ }
+ return Integer.parseInt(stringValue);
+ }
private void modifyColumnFamilyDescriptor(HColumnDescriptor hcd, Map<String,Object> props) throws SQLException {
for (Entry<String, Object> entry : props.entrySet()) {
String propName = entry.getKey();
Object value = entry.getValue();
- hcd.setValue(propName, value == null ? null : value.toString());
+ setHColumnDescriptorValue(hcd, propName, value);
}
}
@@ -622,8 +685,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
new HTableDescriptor(TableName.valueOf(tableName));
for (Entry<String,Object> entry : tableProps.entrySet()) {
String key = entry.getKey();
- Object value = entry.getValue();
- tableDescriptor.setValue(key, value == null ? null : value.toString());
+ if (!TableProperty.isPhoenixTableProperty(key)) {
+ Object value = entry.getValue();
+ tableDescriptor.setValue(key, value == null ? null : value.toString());
+ }
}
if (families.isEmpty()) {
if (tableType != PTableType.VIEW) {
@@ -649,16 +714,20 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
tableDescriptor.addFamily(columnDescriptor);
} else {
if (tableType != PTableType.VIEW) {
- modifyColumnFamilyDescriptor(tableDescriptor.getFamily(familyByte), family);
+ HColumnDescriptor columnDescriptor = tableDescriptor.getFamily(familyByte);
+ if (columnDescriptor == null) {
+ throw new IllegalArgumentException("Unable to find column descriptor with family name " + Bytes.toString(family.getFirst()));
+ }
+ modifyColumnFamilyDescriptor(columnDescriptor, family.getSecond());
}
}
}
}
- addCoprocessors(tableName, tableDescriptor, tableType);
+ addCoprocessors(tableName, tableDescriptor, tableType, tableProps);
return tableDescriptor;
}
- private void addCoprocessors(byte[] tableName, HTableDescriptor descriptor, PTableType tableType) throws SQLException {
+ private void addCoprocessors(byte[] tableName, HTableDescriptor descriptor, PTableType tableType, Map<String,Object> tableProps) throws SQLException {
// The phoenix jar must be available on HBase classpath
int priority = props.getInt(QueryServices.COPROCESSOR_PRIORITY_ATTRIB, QueryServicesOptions.DEFAULT_COPROCESSOR_PRIORITY);
try {
@@ -674,17 +743,35 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (!descriptor.hasCoprocessor(ServerCachingEndpointImpl.class.getName())) {
descriptor.addCoprocessor(ServerCachingEndpointImpl.class.getName(), null, priority, null);
}
+ boolean isTransactional =
+ Boolean.TRUE.equals(tableProps.get(TableProperty.TRANSACTIONAL.name())) ||
+ Boolean.TRUE.equals(tableProps.get(TxConstants.READ_NON_TX_DATA)); // For ALTER TABLE
// TODO: better encapsulation for this
// Since indexes can't have indexes, don't install our indexing coprocessor for indexes.
// Also don't install on the SYSTEM.CATALOG and SYSTEM.STATS table because we use
// all-or-none mutate class which break when this coprocessor is installed (PHOENIX-1318).
if ((tableType != PTableType.INDEX && tableType != PTableType.VIEW)
&& !SchemaUtil.isMetaTable(tableName)
- && !SchemaUtil.isStatsTable(tableName)
- && !descriptor.hasCoprocessor(Indexer.class.getName())) {
- Map<String, String> opts = Maps.newHashMapWithExpectedSize(1);
- opts.put(NonTxIndexBuilder.CODEC_CLASS_NAME_KEY, PhoenixIndexCodec.class.getName());
- Indexer.enableIndexing(descriptor, PhoenixIndexBuilder.class, opts, priority);
+ && !SchemaUtil.isStatsTable(tableName)) {
+ if (isTransactional) {
+ if (!descriptor.hasCoprocessor(PhoenixTransactionalIndexer.class.getName())) {
+ descriptor.addCoprocessor(PhoenixTransactionalIndexer.class.getName(), null, priority, null);
+ }
+ // For alter table, remove non transactional index coprocessor
+ if (descriptor.hasCoprocessor(Indexer.class.getName())) {
+ descriptor.removeCoprocessor(Indexer.class.getName());
+ }
+ } else {
+ if (!descriptor.hasCoprocessor(Indexer.class.getName())) {
+ // If exception on alter table to transition back to non transactional
+ if (descriptor.hasCoprocessor(PhoenixTransactionalIndexer.class.getName())) {
+ descriptor.removeCoprocessor(PhoenixTransactionalIndexer.class.getName());
+ }
+ Map<String, String> opts = Maps.newHashMapWithExpectedSize(1);
+ opts.put(NonTxIndexBuilder.CODEC_CLASS_NAME_KEY, PhoenixIndexCodec.class.getName());
+ Indexer.enableIndexing(descriptor, PhoenixIndexBuilder.class, opts, priority);
+ }
+ }
}
if (SchemaUtil.isStatsTable(tableName) && !descriptor.hasCoprocessor(MultiRowMutationEndpoint.class.getName())) {
descriptor.addCoprocessor(MultiRowMutationEndpoint.class.getName(),
@@ -722,6 +809,17 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
descriptor.addCoprocessor(SequenceRegionObserver.class.getName(), null, priority, null);
}
}
+
+ if (isTransactional) {
+ if (!descriptor.hasCoprocessor(TransactionProcessor.class.getName())) {
+ descriptor.addCoprocessor(TransactionProcessor.class.getName(), null, priority - 10, null);
+ }
+ } else {
+ // If exception on alter table to transition back to non transactional
+ if (descriptor.hasCoprocessor(TransactionProcessor.class.getName())) {
+ descriptor.removeCoprocessor(TransactionProcessor.class.getName());
+ }
+ }
} catch (IOException e) {
throw ServerUtil.parseServerException(e);
}
@@ -879,8 +977,26 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
checkClientServerCompatibility();
}
- if (!modifyExistingMetaData || existingDesc.equals(newDesc)) {
- return existingDesc;
+ if (!modifyExistingMetaData) {
+ return existingDesc; // Caller already knows that no metadata was changed
+ }
+ boolean willBeTx = Boolean.TRUE.equals(props.get(TableProperty.TRANSACTIONAL.name()));
+ // If mapping an existing table as transactional, set property so that existing
+ // data is correctly read.
+ if (willBeTx) {
+ newDesc.setValue(TxConstants.READ_NON_TX_DATA, Boolean.TRUE.toString());
+ } else {
+ // If we think we're creating a non transactional table when it's already
+ // transactional, don't allow.
+ if (existingDesc.hasCoprocessor(TransactionProcessor.class.getName())) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.TX_MAY_NOT_SWITCH_TO_NON_TX)
+ .setSchemaName(SchemaUtil.getSchemaNameFromFullName(tableName))
+ .setTableName(SchemaUtil.getTableNameFromFullName(tableName)).build().buildException();
+ }
+ newDesc.remove(TxConstants.READ_NON_TX_DATA);
+ }
+ if (existingDesc.equals(newDesc)) {
+ return null; // Indicate that no metadata was changed
}
modifyTable(tableName, newDesc, true);
@@ -987,7 +1103,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
buf.append(name);
buf.append(';');
}
- hasInvalidIndexConfiguration |= isInvalidMutableIndexConfig(result.getValue());
+ isMutableIndexWALCodecInstalled &= !isInvalidMutableIndexConfig(result.getValue());
if (minHBaseVersion > MetaDataUtil.decodeHBaseVersion(result.getValue())) {
minHBaseVersion = MetaDataUtil.decodeHBaseVersion(result.getValue());
}
@@ -1096,7 +1212,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
try {
synchronized (latestMetaDataLock) {
throwConnectionClosedIfNullMetaData();
- table = latestMetaData.getTable(new PTableKey(PName.EMPTY_NAME, parentTableName));
+ table = latestMetaData.getTableRef(new PTableKey(PName.EMPTY_NAME, parentTableName)).getTable();
latestMetaDataLock.notifyAll();
}
if (table.getTimeStamp() >= timestamp) { // Table in cache is newer than client timestamp which shouldn't be the case
@@ -1271,7 +1387,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
builder.addTableMetadataMutations(mp.toByteString());
}
builder.setClientVersion(VersionUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER));
- instance.createTable(controller, builder.build(), rpcCallback);
+ CreateTableRequest build = builder.build();
+ instance.createTable(controller, build, rpcCallback);
if(controller.getFailedOn() != null) {
throw controller.getFailedOn();
}
@@ -1460,7 +1577,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (metadata == null) {
throwConnectionClosedException();
}
- table = metadata.getTable(new PTableKey(tenantId, name));
+ table = metadata.getTableRef(new PTableKey(tenantId, name)).getTable();
if (table.getTimeStamp() >= timestamp) { // Table in cache is newer than client timestamp which shouldn't be the case
throw new TableNotFoundException(table.getSchemaName().getString(), table.getTableName().getString());
}
@@ -1505,22 +1622,213 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
@Override
public MetaDataMutationResult addColumn(final List<Mutation> tableMetaData, PTable table, Map<String, List<Pair<String,Object>>> stmtProperties, Set<String> colFamiliesForPColumnsToBeAdded) throws SQLException {
- Map<String, Object> tableProps = new HashMap<String, Object>();
- HTableDescriptor tableDescriptor = separateAndValidateProperties(table, stmtProperties, colFamiliesForPColumnsToBeAdded, tableProps);
- SQLException sqlE = null;
+ List<Pair<byte[], Map<String, Object>>> families = new ArrayList<>(stmtProperties.size());
+ Map<String, Object> tableProps = new HashMap<String, Object>();
+ Set<HTableDescriptor> tableDescriptors = Collections.emptySet();
+ Set<HTableDescriptor> origTableDescriptors = Collections.emptySet();
+ boolean nonTxToTx = false;
+ Pair<HTableDescriptor,HTableDescriptor> tableDescriptorPair = separateAndValidateProperties(table, stmtProperties, colFamiliesForPColumnsToBeAdded, families, tableProps);
+ HTableDescriptor tableDescriptor = tableDescriptorPair.getSecond();
+ HTableDescriptor origTableDescriptor = tableDescriptorPair.getFirst();
if (tableDescriptor != null) {
- try {
- boolean modifyHTable = true;
- if (table.getType() == PTableType.VIEW) {
- boolean canViewsAddNewCF = props.getBoolean(QueryServices.ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE,
- QueryServicesOptions.DEFAULT_ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE);
- // When adding a column to a view, base physical table should only be modified when new column families are being added.
- modifyHTable = canViewsAddNewCF && !existingColumnFamiliesForBaseTable(table.getPhysicalName()).containsAll(colFamiliesForPColumnsToBeAdded);
+ tableDescriptors = Sets.newHashSetWithExpectedSize(3 + table.getIndexes().size());
+ origTableDescriptors = Sets.newHashSetWithExpectedSize(3 + table.getIndexes().size());
+ tableDescriptors.add(tableDescriptor);
+ origTableDescriptors.add(origTableDescriptor);
+ nonTxToTx = Boolean.TRUE.equals(tableProps.get(TxConstants.READ_NON_TX_DATA));
+ /*
+ * If the table was transitioned from non transactional to transactional, we need
+ * to also transition the index tables.
+ */
+ if (nonTxToTx) {
+ updateDescriptorForTx(table, tableProps, tableDescriptor, Boolean.TRUE.toString(), tableDescriptors, origTableDescriptors);
+ }
+ }
+
+ boolean success = false;
+ boolean metaDataUpdated = !tableDescriptors.isEmpty();
+ boolean pollingNeeded = !(!tableProps.isEmpty() && families.isEmpty() && colFamiliesForPColumnsToBeAdded.isEmpty());
+ MetaDataMutationResult result = null;
+ try {
+ boolean modifyHTable = true;
+ if (table.getType() == PTableType.VIEW) {
+ boolean canViewsAddNewCF = props.getBoolean(QueryServices.ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE,
+ QueryServicesOptions.DEFAULT_ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE);
+ // When adding a column to a view, base physical table should only be modified when new column families are being added.
+ modifyHTable = canViewsAddNewCF && !existingColumnFamiliesForBaseTable(table.getPhysicalName()).containsAll(colFamiliesForPColumnsToBeAdded);
+ }
+ if (modifyHTable) {
+ sendHBaseMetaData(tableDescriptors, pollingNeeded);
+ }
+
+ // Special case for call during drop table to ensure that the empty column family exists.
+ // In this, case we only include the table header row, as until we add schemaBytes and tableBytes
+ // as args to this function, we have no way of getting them in this case.
+ // TODO: change to if (tableMetaData.isEmpty()) once we pass through schemaBytes and tableBytes
+ // Also, could be used to update property values on ALTER TABLE t SET prop=xxx
+ if ((tableMetaData.isEmpty()) || (tableMetaData.size() == 1 && tableMetaData.get(0).isEmpty())) {
+ return new MetaDataMutationResult(MutationCode.NO_OP, System.currentTimeMillis(), table);
+ }
+ byte[][] rowKeyMetaData = new byte[3][];
+ PTableType tableType = table.getType();
+
+ Mutation m = tableMetaData.get(0);
+ byte[] rowKey = m.getRow();
+ SchemaUtil.getVarChars(rowKey, rowKeyMetaData);
+ byte[] tenantIdBytes = rowKeyMetaData[PhoenixDatabaseMetaData.TENANT_ID_INDEX];
+ byte[] schemaBytes = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
+ byte[] tableBytes = rowKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
+ byte[] tableKey = SchemaUtil.getTableKey(tenantIdBytes, schemaBytes, tableBytes);
+
+ ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+ result = metaDataCoprocessorExec(tableKey,
+ new Batch.Call<MetaDataService, MetaDataResponse>() {
+ @Override
+ public MetaDataResponse call(MetaDataService instance) throws IOException {
+ ServerRpcController controller = new ServerRpcController();
+ BlockingRpcCallback<MetaDataResponse> rpcCallback =
+ new BlockingRpcCallback<MetaDataResponse>();
+ AddColumnRequest.Builder builder = AddColumnRequest.newBuilder();
+ for (Mutation m : tableMetaData) {
+ MutationProto mp = ProtobufUtil.toProto(m);
+ builder.addTableMetadataMutations(mp.toByteString());
+ }
+ builder.setClientVersion(VersionUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER));
+ instance.addColumn(controller, builder.build(), rpcCallback);
+ if(controller.getFailedOn() != null) {
+ throw controller.getFailedOn();
+ }
+ return rpcCallback.get();
+ }
+ });
+
+ if (result.getMutationCode() == MutationCode.COLUMN_NOT_FOUND || result.getMutationCode() == MutationCode.TABLE_ALREADY_EXISTS) { // Success
+ success = true;
+ // Flush the table if transitioning DISABLE_WAL from TRUE to FALSE
+ if ( MetaDataUtil.getMutationValue(m,PhoenixDatabaseMetaData.DISABLE_WAL_BYTES, kvBuilder, ptr)
+ && Boolean.FALSE.equals(PBoolean.INSTANCE.toObject(ptr))) {
+ flushTable(table.getPhysicalName().getBytes());
+ }
+
+ if (tableType == PTableType.TABLE) {
+ // If we're changing MULTI_TENANT to true or false, create or drop the view index table
+ if (MetaDataUtil.getMutationValue(m, PhoenixDatabaseMetaData.MULTI_TENANT_BYTES, kvBuilder, ptr)){
+ long timestamp = MetaDataUtil.getClientTimeStamp(m);
+ if (Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(ptr.get(), ptr.getOffset(), ptr.getLength()))) {
+ this.ensureViewIndexTableCreated(table, timestamp);
+ } else {
+ this.ensureViewIndexTableDropped(table.getPhysicalName().getBytes(), timestamp);
+ }
+ }
}
- boolean pollingNotNeeded = (!tableProps.isEmpty() && !existingColumnFamilies(table).containsAll(colFamiliesForPColumnsToBeAdded));
- if (modifyHTable) {
- modifyTable(table.getPhysicalName().getBytes(), tableDescriptor, !pollingNotNeeded);
+ }
+ } finally {
+ // If we weren't successful with our metadata update
+ // and we've already pushed the HBase metadata changes to the server
+ // and we've tried to go from non transactional to transactional
+ // then we must undo the metadata change otherwise the table will
+ // no longer function correctly.
+ // Note that if this fails, we're in a corrupt state.
+ if (!success && metaDataUpdated && nonTxToTx) {
+ sendHBaseMetaData(origTableDescriptors, pollingNeeded);
+ }
+ }
+ return result;
+ }
+ private void updateDescriptorForTx(PTable table, Map<String, Object> tableProps, HTableDescriptor tableDescriptor,
+ String txValue, Set<HTableDescriptor> descriptorsToUpdate, Set<HTableDescriptor> origDescriptors) throws SQLException {
+ HBaseAdmin admin = null;
+ byte[] physicalTableName = table.getPhysicalName().getBytes();
+ try {
+ admin = new HBaseAdmin(config);
+ setTransactional(tableDescriptor, table.getType(), txValue, tableProps);
+ Map<String, Object> indexTableProps;
+ if (txValue == null) {
+ indexTableProps = Collections.<String,Object>emptyMap();
+ } else {
+ indexTableProps = Maps.newHashMapWithExpectedSize(1);
+ indexTableProps.put(TxConstants.READ_NON_TX_DATA, Boolean.valueOf(txValue));
+ }
+ for (PTable index : table.getIndexes()) {
+ HTableDescriptor indexDescriptor = admin.getTableDescriptor(index.getPhysicalName().getBytes());
+ origDescriptors.add(indexDescriptor);
+ indexDescriptor = new HTableDescriptor(indexDescriptor);
+ descriptorsToUpdate.add(indexDescriptor);
+ if (index.getColumnFamilies().isEmpty()) {
+ byte[] dataFamilyName = SchemaUtil.getEmptyColumnFamily(table);
+ byte[] indexFamilyName = SchemaUtil.getEmptyColumnFamily(index);
+ HColumnDescriptor indexColDescriptor = indexDescriptor.getFamily(indexFamilyName);
+ HColumnDescriptor tableColDescriptor = tableDescriptor.getFamily(dataFamilyName);
+ indexColDescriptor.setMaxVersions(tableColDescriptor.getMaxVersions());
+ indexColDescriptor.setValue(TxConstants.PROPERTY_TTL, tableColDescriptor.getValue(TxConstants.PROPERTY_TTL));
+ } else {
+ for (PColumnFamily family : index.getColumnFamilies()) {
+ byte[] familyName = family.getName().getBytes();
+ indexDescriptor.getFamily(familyName).setMaxVersions(tableDescriptor.getFamily(familyName).getMaxVersions());
+ HColumnDescriptor indexColDescriptor = indexDescriptor.getFamily(familyName);
+ HColumnDescriptor tableColDescriptor = tableDescriptor.getFamily(familyName);
+ indexColDescriptor.setMaxVersions(tableColDescriptor.getMaxVersions());
+ indexColDescriptor.setValue(TxConstants.PROPERTY_TTL, tableColDescriptor.getValue(TxConstants.PROPERTY_TTL));
+ }
}
+ setTransactional(indexDescriptor, index.getType(), txValue, indexTableProps);
+ }
+ try {
+ HTableDescriptor indexDescriptor = admin.getTableDescriptor(MetaDataUtil.getViewIndexPhysicalName(physicalTableName));
+ origDescriptors.add(indexDescriptor);
+ indexDescriptor = new HTableDescriptor(indexDescriptor);
+ descriptorsToUpdate.add(indexDescriptor);
+ setSharedIndexMaxVersion(table, tableDescriptor, indexDescriptor);
+ setTransactional(indexDescriptor, PTableType.INDEX, txValue, indexTableProps);
+ } catch (org.apache.hadoop.hbase.TableNotFoundException ignore) {
+ // Ignore, as we may never have created a view index table
+ }
+ try {
+ HTableDescriptor indexDescriptor = admin.getTableDescriptor(MetaDataUtil.getLocalIndexPhysicalName(physicalTableName));
+ origDescriptors.add(indexDescriptor);
+ indexDescriptor = new HTableDescriptor(indexDescriptor);
+ descriptorsToUpdate.add(indexDescriptor);
+ setSharedIndexMaxVersion(table, tableDescriptor, indexDescriptor);
+ setTransactional(indexDescriptor, PTableType.INDEX, txValue, indexTableProps);
+ } catch (org.apache.hadoop.hbase.TableNotFoundException ignore) {
+ // Ignore, as we may never have created a view index table
+ }
+ } catch (IOException e) {
+ throw ServerUtil.parseServerException(e);
+ } finally {
+ try {
+ if (admin != null) admin.close();
+ } catch (IOException e) {
+ logger.warn("Could not close admin",e);
+ }
+ }
+ }
+ private void setSharedIndexMaxVersion(PTable table, HTableDescriptor tableDescriptor,
+ HTableDescriptor indexDescriptor) {
+ if (table.getColumnFamilies().isEmpty()) {
+ byte[] familyName = SchemaUtil.getEmptyColumnFamily(table);
+ HColumnDescriptor indexColDescriptor = indexDescriptor.getFamily(familyName);
+ HColumnDescriptor tableColDescriptor = tableDescriptor.getFamily(familyName);
+ indexColDescriptor.setMaxVersions(tableColDescriptor.getMaxVersions());
+ indexColDescriptor.setValue(TxConstants.PROPERTY_TTL, tableColDescriptor.getValue(TxConstants.PROPERTY_TTL));
+ } else {
+ for (PColumnFamily family : table.getColumnFamilies()) {
+ byte[] familyName = family.getName().getBytes();
+ HColumnDescriptor indexColDescriptor = indexDescriptor.getFamily(familyName);
+ if (indexColDescriptor != null) {
+ HColumnDescriptor tableColDescriptor = tableDescriptor.getFamily(familyName);
+ indexColDescriptor.setMaxVersions(tableColDescriptor.getMaxVersions());
+ indexColDescriptor.setValue(TxConstants.PROPERTY_TTL, tableColDescriptor.getValue(TxConstants.PROPERTY_TTL));
+ }
+ }
+ }
+ }
+
+ private void sendHBaseMetaData(Set<HTableDescriptor> tableDescriptors, boolean pollingNeeded) throws SQLException {
+ SQLException sqlE = null;
+ for (HTableDescriptor descriptor : tableDescriptors) {
+ try {
+ modifyTable(descriptor.getName(), descriptor, pollingNeeded);
} catch (IOException e) {
sqlE = ServerUtil.parseServerException(e);
} catch (InterruptedException e) {
@@ -1535,76 +1843,26 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
}
-
- // Special case for call during drop table to ensure that the empty column family exists.
- // In this, case we only include the table header row, as until we add schemaBytes and tableBytes
- // as args to this function, we have no way of getting them in this case.
- // TODO: change to if (tableMetaData.isEmpty()) once we pass through schemaBytes and tableBytes
- // Also, could be used to update property values on ALTER TABLE t SET prop=xxx
- if ((tableMetaData.isEmpty()) || (tableMetaData.size() == 1 && tableMetaData.get(0).isEmpty())) {
- return new MetaDataMutationResult(MutationCode.NO_OP, System.currentTimeMillis(), table);
- }
- byte[][] rowKeyMetaData = new byte[3][];
- PTableType tableType = table.getType();
-
- Mutation m = tableMetaData.get(0);
- byte[] rowKey = m.getRow();
- SchemaUtil.getVarChars(rowKey, rowKeyMetaData);
- byte[] tenantIdBytes = rowKeyMetaData[PhoenixDatabaseMetaData.TENANT_ID_INDEX];
- byte[] schemaBytes = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
- byte[] tableBytes = rowKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
- byte[] tableKey = SchemaUtil.getTableKey(tenantIdBytes, schemaBytes, tableBytes);
-
- ImmutableBytesWritable ptr = new ImmutableBytesWritable();
- MetaDataMutationResult result = metaDataCoprocessorExec(tableKey,
- new Batch.Call<MetaDataService, MetaDataResponse>() {
- @Override
- public MetaDataResponse call(MetaDataService instance) throws IOException {
- ServerRpcController controller = new ServerRpcController();
- BlockingRpcCallback<MetaDataResponse> rpcCallback =
- new BlockingRpcCallback<MetaDataResponse>();
- AddColumnRequest.Builder builder = AddColumnRequest.newBuilder();
- for (Mutation m : tableMetaData) {
- MutationProto mp = ProtobufUtil.toProto(m);
- builder.addTableMetadataMutations(mp.toByteString());
- }
- builder.setClientVersion(VersionUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER));
- instance.addColumn(controller, builder.build(), rpcCallback);
- if(controller.getFailedOn() != null) {
- throw controller.getFailedOn();
- }
- return rpcCallback.get();
- }
- });
-
- if (result.getMutationCode() == MutationCode.COLUMN_NOT_FOUND) { // Success
- // Flush the table if transitioning DISABLE_WAL from TRUE to FALSE
- if ( MetaDataUtil.getMutationValue(m,PhoenixDatabaseMetaData.DISABLE_WAL_BYTES, kvBuilder, ptr)
- && Boolean.FALSE.equals(PBoolean.INSTANCE.toObject(ptr))) {
- flushTable(table.getPhysicalName().getBytes());
- }
-
- if (tableType == PTableType.TABLE) {
- // If we're changing MULTI_TENANT to true or false, create or drop the view index table
- if (MetaDataUtil.getMutationValue(m, PhoenixDatabaseMetaData.MULTI_TENANT_BYTES, kvBuilder, ptr)){
- long timestamp = MetaDataUtil.getClientTimeStamp(m);
- if (Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(ptr.get(), ptr.getOffset(), ptr.getLength()))) {
- this.ensureViewIndexTableCreated(table, timestamp);
- } else {
- this.ensureViewIndexTableDropped(table.getPhysicalName().getBytes(), timestamp);
- }
- }
- }
+ }
+ private void setTransactional(HTableDescriptor tableDescriptor, PTableType tableType, String txValue, Map<String, Object> tableProps) throws SQLException {
+ if (txValue == null) {
+ tableDescriptor.remove(TxConstants.READ_NON_TX_DATA);
+ } else {
+ tableDescriptor.setValue(TxConstants.READ_NON_TX_DATA, txValue);
}
- return result;
+ this.addCoprocessors(tableDescriptor.getName(), tableDescriptor, tableType, tableProps);
}
- private HTableDescriptor separateAndValidateProperties(PTable table, Map<String, List<Pair<String, Object>>> properties, Set<String> colFamiliesForPColumnsToBeAdded, Map<String, Object> tableProps) throws SQLException {
+ private Pair<HTableDescriptor,HTableDescriptor> separateAndValidateProperties(PTable table, Map<String, List<Pair<String, Object>>> properties, Set<String> colFamiliesForPColumnsToBeAdded, List<Pair<byte[], Map<String, Object>>> families, Map<String, Object> tableProps) throws SQLException {
Map<String, Map<String, Object>> stmtFamiliesPropsMap = new HashMap<>(properties.size());
Map<String,Object> commonFamilyProps = new HashMap<>();
- boolean addingColumns = colFamiliesForPColumnsToBeAdded != null && colFamiliesForPColumnsToBeAdded.size() > 0;
+ boolean addingColumns = colFamiliesForPColumnsToBeAdded != null && !colFamiliesForPColumnsToBeAdded.isEmpty();
HashSet<String> existingColumnFamilies = existingColumnFamilies(table);
Map<String, Map<String, Object>> allFamiliesProps = new HashMap<>(existingColumnFamilies.size());
+ boolean isTransactional = table.isTransactional();
+ boolean willBeTransactional = false;
+ boolean isOrWillBeTransactional = isTransactional;
+ Integer newTTL = null;
for (String family : properties.keySet()) {
List<Pair<String, Object>> propsList = properties.get(family);
if (propsList != null && propsList.size() > 0) {
@@ -1630,22 +1888,26 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (TableProperty.isPhoenixTableProperty(propName)) {
TableProperty.valueOf(propName).validate(true, !family.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY), table.getType());
if (propName.equals(TTL)) {
+ newTTL = ((Number)prop.getSecond()).intValue();
// Even though TTL is really a HColumnProperty we treat it specially.
// We enforce that all column families have the same TTL.
commonFamilyProps.put(propName, prop.getSecond());
+ } else if (propName.equals(PhoenixDatabaseMetaData.TRANSACTIONAL) && Boolean.TRUE.equals(propValue)) {
+ willBeTransactional = isOrWillBeTransactional = true;
+ tableProps.put(TxConstants.READ_NON_TX_DATA, propValue);
}
} else {
if (isHColumnProperty(propName)) {
if (family.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY)) {
- commonFamilyProps.put(propName, prop.getSecond());
+ commonFamilyProps.put(propName, propValue);
} else {
- colFamilyPropsMap.put(propName, prop.getSecond());
+ colFamilyPropsMap.put(propName, propValue);
}
} else {
// invalid property - neither of HTableProp, HColumnProp or PhoenixTableProp
// FIXME: This isn't getting triggered as currently a property gets evaluated
// as HTableProp if its neither HColumnProp or PhoenixTableProp.
- throw new SQLExceptionInfo.Builder(SQLExceptionCode.SET_UNSUPPORTED_PROP_ON_ALTER_TABLE)
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_PROPERTY)
.setMessage("Column Family: " + family + ", Property: " + propName).build()
.buildException();
}
@@ -1728,16 +1990,17 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
allFamiliesProps.put(Bytes.toString(table.getDefaultFamilyName() == null ? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES : table.getDefaultFamilyName().getBytes() ), commonFamilyProps);
}
-
// Views are not allowed to have any of these properties.
if (table.getType() == PTableType.VIEW && (!stmtFamiliesPropsMap.isEmpty() || !commonFamilyProps.isEmpty() || !tableProps.isEmpty())) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.VIEW_WITH_PROPERTIES).build()
.buildException();
}
+
HTableDescriptor newTableDescriptor = null;
+ HTableDescriptor origTableDescriptor = null;
if (!allFamiliesProps.isEmpty() || !tableProps.isEmpty()) {
byte[] tableNameBytes = Bytes.toBytes(table.getPhysicalName().getString());
- HTableDescriptor existingTableDescriptor = getTableDescriptor(tableNameBytes);
+ HTableDescriptor existingTableDescriptor = origTableDescriptor = getTableDescriptor(tableNameBytes);
newTableDescriptor = new HTableDescriptor(existingTableDescriptor);
if (!tableProps.isEmpty()) {
// add all the table properties to the existing table descriptor
@@ -1747,21 +2010,97 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
if (addingColumns) {
// Make sure that all the CFs of the table have the same TTL as the empty CF.
- setTTLToEmptyCFTTL(allFamiliesProps, table, newTableDescriptor);
+ setTTLForNewCFs(allFamiliesProps, table, newTableDescriptor, newTTL);
+ }
+ // Set TTL on all table column families, even if they're not referenced here
+ if (newTTL != null) {
+ for (PColumnFamily family : table.getColumnFamilies()) {
+ if (!allFamiliesProps.containsKey(family.getName().getString())) {
+ Map<String,Object> familyProps = Maps.newHashMapWithExpectedSize(1);
+ familyProps.put(TTL, newTTL);
+ allFamiliesProps.put(family.getName().getString(), familyProps);
+ }
+ }
+ }
+ Integer defaultTxMaxVersions = null;
+ if (isOrWillBeTransactional) {
+ // Calculate default for max versions
+ Map<String, Object> emptyFamilyProps = allFamiliesProps.get(SchemaUtil.getEmptyColumnFamilyAsString(table));
+ if (emptyFamilyProps != null) {
+ defaultTxMaxVersions = (Integer)emptyFamilyProps.get(HConstants.VERSIONS);
+ }
+ if (defaultTxMaxVersions == null) {
+ if (isTransactional) {
+ defaultTxMaxVersions = newTableDescriptor.getFamily(SchemaUtil.getEmptyColumnFamily(table)).getMaxVersions();
+ } else {
+ defaultTxMaxVersions =
+ this.getProps().getInt(
+ QueryServices.MAX_VERSIONS_TRANSACTIONAL_ATTRIB,
+ QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL);
+ }
+ }
+ if (willBeTransactional) {
+ // Set VERSIONS for all column families when transitioning to transactional
+ for (PColumnFamily family : table.getColumnFamilies()) {
+ if (!allFamiliesProps.containsKey(family.getName().getString())) {
+ Map<String,Object> familyProps = Maps.newHashMapWithExpectedSize(1);
+ familyProps.put(HConstants.VERSIONS, defaultTxMaxVersions);
+ allFamiliesProps.put(family.getName().getString(), familyProps);
+ }
+ }
+ }
+ }
+ // Set Tephra's TTL property based on HBase property if we're
+ // transitioning to become transactional or setting TTL on
+ // an already transactional table.
+ if (isOrWillBeTransactional) {
+ int ttl = getTTL(table, newTableDescriptor, newTTL);
+ if (ttl != HColumnDescriptor.DEFAULT_TTL) {
+ for (Map.Entry<String, Map<String, Object>> entry : allFamiliesProps.entrySet()) {
+ Map<String, Object> props = entry.getValue();
+ if (props == null) {
+ props = new HashMap<String, Object>();
+ }
+ props.put(TxConstants.PROPERTY_TTL, ttl);
+ // Remove HBase TTL if we're not transitioning an existing table to become transactional
+ // or if the existing transactional table wasn't originally non transactional.
+ if (!willBeTransactional && !Boolean.valueOf(newTableDescriptor.getValue(TxConstants.READ_NON_TX_DATA))) {
+ props.remove(TTL);
+ }
+ }
+ }
}
for (Entry<String, Map<String, Object>> entry : allFamiliesProps.entrySet()) {
- byte[] cf = entry.getKey().getBytes();
+ Map<String,Object> familyProps = entry.getValue();
+ if (isOrWillBeTransactional) {
+ if (!familyProps.containsKey(HConstants.VERSIONS)) {
+ familyProps.put(HConstants.VERSIONS, defaultTxMaxVersions);
+ }
+ }
+ byte[] cf = Bytes.toBytes(entry.getKey());
HColumnDescriptor colDescriptor = newTableDescriptor.getFamily(cf);
if (colDescriptor == null) {
// new column family
- colDescriptor = generateColumnFamilyDescriptor(new Pair<>(cf, entry.getValue()), table.getType());
+ colDescriptor = generateColumnFamilyDescriptor(new Pair<>(cf, familyProps), table.getType());
newTableDescriptor.addFamily(colDescriptor);
} else {
- modifyColumnFamilyDescriptor(colDescriptor, entry.getValue());
+ modifyColumnFamilyDescriptor(colDescriptor, familyProps);
+ }
+ if (isOrWillBeTransactional) {
+ checkTransactionalVersionsValue(colDescriptor);
}
}
}
- return newTableDescriptor;
+ return new Pair<>(origTableDescriptor, newTableDescriptor);
+ }
+
+ private void checkTransactionalVersionsValue(HColumnDescriptor colDescriptor) throws SQLException {
+ int maxVersions = colDescriptor.getMaxVersions();
+ if (maxVersions <= 1) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE)
+ .setFamilyName(colDescriptor.getNameAsString())
+ .build().buildException();
+ }
}
private boolean isHColumnProperty(String propName) {
@@ -1775,7 +2114,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private HashSet<String> existingColumnFamiliesForBaseTable(PName baseTableName) throws TableNotFoundException {
synchronized (latestMetaDataLock) {
throwConnectionClosedIfNullMetaData();
- PTable table = latestMetaData.getTable(new PTableKey(null, baseTableName.getString()));
+ PTable table = latestMetaData.getTableRef(new PTableKey(null, baseTableName.getString())).getTable();
latestMetaDataLock.notifyAll();
return existingColumnFamilies(table);
}
@@ -1790,20 +2129,23 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
return cfNames;
}
- private int getTTLForEmptyCf(byte[] emptyCf, byte[] tableNameBytes, HTableDescriptor tableDescriptor) throws SQLException {
- if (tableDescriptor == null) {
- tableDescriptor = getTableDescriptor(tableNameBytes);
- }
- return tableDescriptor.getFamily(emptyCf).getTimeToLive();
+ private static int getTTL(PTable table, HTableDescriptor tableDesc, Integer newTTL) throws SQLException {
+ // If we're setting TTL now, then use that value. Otherwise, use empty column family value
+ int ttl = newTTL != null ? newTTL
+ : tableDesc.getFamily(SchemaUtil.getEmptyColumnFamily(table)).getTimeToLive();
+ return ttl;
}
- private void setTTLToEmptyCFTTL(Map<String, Map<String, Object>> familyProps, PTable table,
- HTableDescriptor tableDesc) throws SQLException {
+ private static void setTTLForNewCFs(Map<String, Map<String, Object>> familyProps, PTable table,
+ HTableDescriptor tableDesc, Integer newTTL) throws SQLException {
if (!familyProps.isEmpty()) {
- int emptyCFTTL = getTTLForEmptyCf(SchemaUtil.getEmptyColumnFamily(table), table.getPhysicalName().getBytes(), tableDesc);
- for (String family : familyProps.keySet()) {
- Map<String, Object> props = familyProps.get(family) != null ? familyProps.get(family) : new HashMap<String, Object>();
- props.put(TTL, emptyCFTTL);
+ int ttl = getTTL(table, tableDesc, newTTL);
+ for (Map.Entry<String, Map<String, Object>> entry : familyProps.entrySet()) {
+ Map<String, Object> props = entry.getValue();
+ if (props == null) {
+ props = new HashMap<String, Object>();
+ }
+ props.put(TTL, ttl);
}
}
}
@@ -1860,7 +2202,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
Properties props = PropertiesUtil.deepCopy(oldMetaConnection.getClientInfo());
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp));
// Cannot go through DriverManager or you end up in an infinite loop because it'll call init again
- PhoenixConnection metaConnection = new PhoenixConnection(this, oldMetaConnection.getURL(), props, oldMetaConnection.getMetaDataCache());
+ PhoenixConnection metaConnection = new PhoenixConnection(oldMetaConnection, this, props);
SQLException sqlE = null;
try {
metaConnection.createStatement().executeUpdate("ALTER TABLE " + tableName + " ADD " + (addIfNotExists ? " IF NOT EXISTS " : "") + columns );
@@ -1938,7 +2280,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
String columnsToAdd = "";
if(currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0) {
// We know that we always need to add the STORE_NULLS column for 4.3 release
- columnsToAdd = ", " + PhoenixDatabaseMetaData.STORE_NULLS + " " + PBoolean.INSTANCE.getSqlTypeName();
+ columnsToAdd += "," + PhoenixDatabaseMetaData.STORE_NULLS + " " + PBoolean.INSTANCE.getSqlTypeName();
HBaseAdmin admin = null;
try {
admin = getAdmin();
@@ -2020,6 +2362,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
metaConnection = addColumn(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, columnsToAdd, false);
}
+ if(currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) {
+ columnsToAdd = PhoenixDatabaseMetaData.TRANSACTIONAL + " " + PBoolean.INSTANCE.getSqlTypeName();
+ metaConnection = addColumn(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0, columnsToAdd, false);
+ }
}
int nSaltBuckets = ConnectionQueryServicesImpl.this.props.getInt(QueryServices.SEQUENCE_SALT_BUCKETS_ATTRIB,
@@ -2138,8 +2485,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
@Override
- public boolean hasInvalidIndexConfiguration() {
- return hasInvalidIndexConfiguration;
+ public boolean isMutableIndexWALCodecInstalled() {
+ return isMutableIndexWALCodecInstalled;
}
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 87fb88e..77e4ba9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -24,9 +24,11 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
@@ -81,6 +83,10 @@ import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.SequenceUtil;
+import co.cask.tephra.TransactionManager;
+import co.cask.tephra.TransactionSystemClient;
+import co.cask.tephra.inmemory.InMemoryTxSystemClient;
+
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -99,18 +105,37 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
private PMetaData metaData;
private final Map<SequenceKey, SequenceInfo> sequenceMap = Maps.newHashMap();
private final String userName;
+ private final TransactionSystemClient txSystemClient;
private KeyValueBuilder kvBuilder;
private volatile boolean initialized;
private volatile SQLException initializationException;
private final Map<String, List<HRegionLocation>> tableSplits = Maps.newHashMap();
- public ConnectionlessQueryServicesImpl(QueryServices queryServices, ConnectionInfo connInfo) {
- super(queryServices);
+ public ConnectionlessQueryServicesImpl(QueryServices services, ConnectionInfo connInfo, Properties info) {
+ super(services);
userName = connInfo.getPrincipal();
metaData = newEmptyMetaData();
// Use KeyValueBuilder that builds real KeyValues, as our test utils require this
this.kvBuilder = GenericKeyValueBuilder.INSTANCE;
+ Configuration config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration();
+ for (Entry<String,String> entry : services.getProps()) {
+ config.set(entry.getKey(), entry.getValue());
+ }
+ if (info != null) {
+ for (Object key : info.keySet()) {
+ config.set((String) key, info.getProperty((String) key));
+ }
+ }
+ for (Entry<String,String> entry : connInfo.asProps()) {
+ config.set(entry.getKey(), entry.getValue());
+ }
+
+ // Without making a copy of the configuration we cons up, we lose some of our properties
+ // on the server side during testing.
+ config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration(config);
+ TransactionManager txnManager = new TransactionManager(config);
+ this.txSystemClient = new InMemoryTxSystemClient(txnManager);
}
private PMetaData newEmptyMetaData() {
@@ -141,14 +166,19 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
}
@Override
- public PMetaData addTable(PTable table) throws SQLException {
- return metaData = metaData.addTable(table);
+ public PMetaData addTable(PTable table, long resolvedTime) throws SQLException {
+ return metaData = metaData.addTable(table, resolvedTime);
+ }
+
+ @Override
+ public PMetaData updateResolvedTimestamp(PTable table, long resolvedTimestamp) throws SQLException {
+ return metaData = metaData.updateResolvedTimestamp(table, resolvedTimestamp);
}
@Override
public PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columns, long tableTimeStamp,
- long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls) throws SQLException {
- return metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls);
+ long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException {
+ return metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime);
}
@Override
@@ -159,8 +189,8 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
@Override
public PMetaData removeColumn(PName tenantId, String tableName, List<PColumn> columnsToRemove, long tableTimeStamp,
- long tableSeqNum) throws SQLException {
- return metaData = metaData.removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum);
+ long tableSeqNum, long resolvedTime) throws SQLException {
+ return metaData = metaData.removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum, resolvedTime);
}
@@ -175,7 +205,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
// to get anything from the server (since we don't have a connection)
try {
String fullTableName = SchemaUtil.getTableName(schemaBytes, tableBytes);
- PTable table = metaData.getTable(new PTableKey(tenantId, fullTableName));
+ PTable table = metaData.getTableRef(new PTableKey(tenantId, fullTableName)).getTable();
return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, 0, table, true);
} catch (TableNotFoundException e) {
return new MetaDataMutationResult(MutationCode.TABLE_NOT_FOUND, 0, null);
@@ -345,7 +375,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
String indexName = Bytes.toString(rowKeyMetadata[PhoenixDatabaseMetaData.TABLE_NAME_INDEX]);
String indexTableName = SchemaUtil.getTableName(schemaName, indexName);
PName tenantId = tenantIdBytes.length == 0 ? null : PNameFactory.newName(tenantIdBytes);
- PTable index = metaData.getTable(new PTableKey(tenantId, indexTableName));
+ PTable index = metaData.getTableRef(new PTableKey(tenantId, indexTableName)).getTable();
index = PTableImpl.makePTable(index,newState == PIndexState.USABLE ? PIndexState.ACTIVE : newState == PIndexState.UNUSABLE ? PIndexState.INACTIVE : newState);
return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, 0, index);
}
@@ -360,8 +390,8 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
}
@Override
- public boolean hasInvalidIndexConfiguration() {
- return false;
+ public boolean isMutableIndexWALCodecInstalled() {
+ return true;
}
@Override
@@ -492,6 +522,10 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
}
@Override
+ public TransactionSystemClient getTransactionSystemClient() {
+ return txSystemClient;
+ }
+
public MetaDataMutationResult createFunction(List<Mutation> functionData, PFunction function, boolean temporary)
throws SQLException {
return new MetaDataMutationResult(MutationCode.FUNCTION_NOT_FOUND, 0l, null);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index 4153652..4952355 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -46,6 +46,8 @@ import org.apache.phoenix.schema.SequenceAllocation;
import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.stats.PTableStats;
+import co.cask.tephra.TransactionSystemClient;
+
public class DelegateConnectionQueryServices extends DelegateQueryServices implements ConnectionQueryServices {
@@ -74,14 +76,19 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
}
@Override
- public PMetaData addTable(PTable table) throws SQLException {
- return getDelegate().addTable(table);
+ public PMetaData addTable(PTable table, long resolvedTime) throws SQLException {
+ return getDelegate().addTable(table, resolvedTime);
+ }
+
+ @Override
+ public PMetaData updateResolvedTimestamp(PTable table, long resolvedTimestamp) throws SQLException {
+ return getDelegate().updateResolvedTimestamp(table, resolvedTimestamp);
}
@Override
public PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columns, long tableTimeStamp,
- long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls) throws SQLException {
- return getDelegate().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls);
+ long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException {
+ return getDelegate().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime);
}
@Override
@@ -92,8 +99,8 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
@Override
public PMetaData removeColumn(PName tenantId, String tableName, List<PColumn> columnsToRemove, long tableTimeStamp,
- long tableSeqNum) throws SQLException {
- return getDelegate().removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum);
+ long tableSeqNum, long resolvedTime) throws SQLException {
+ return getDelegate().removeColumn(tenantId, tableName, columnsToRemove, tableTimeStamp, tableSeqNum, resolvedTime);
}
@Override
@@ -165,8 +172,8 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
}
@Override
- public boolean hasInvalidIndexConfiguration() {
- return getDelegate().hasInvalidIndexConfiguration();
+ public boolean isMutableIndexWALCodecInstalled() {
+ return getDelegate().isMutableIndexWALCodecInstalled();
}
@Override
@@ -254,6 +261,10 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
}
@Override
+ public TransactionSystemClient getTransactionSystemClient() {
+ return getDelegate().getTransactionSystemClient();
+ }
+
public MetaDataMutationResult createFunction(List<Mutation> functionData, PFunction function, boolean temporary)
throws SQLException {
return getDelegate().createFunction(functionData, function, temporary);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java b/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java
index 76e7593..8e7a70d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java
@@ -35,10 +35,11 @@ import org.apache.phoenix.schema.PTable;
* @since 0.1
*/
public interface MetaDataMutated {
- PMetaData addTable(PTable table) throws SQLException;
+ PMetaData addTable(PTable table, long resolvedTime) throws SQLException;
+ PMetaData updateResolvedTimestamp(PTable table, long resolvedTimestamp) throws SQLException;
PMetaData removeTable(PName tenantId, String tableName, String parentTableName, long tableTimeStamp) throws SQLException;
- PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls) throws SQLException;
- PMetaData removeColumn(PName tenantId, String tableName, List<PColumn> columnsToRemove, long tableTimeStamp, long tableSeqNum) throws SQLException;
+ PMetaData addColumn(PName tenantId, String tableName, List<PColumn> columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException;
+ PMetaData removeColumn(PName tenantId, String tableName, List<PColumn> columnsToRemove, long tableTimeStamp, long tableSeqNum, long resolvedTime) throws SQLException;
PMetaData addFunction(PFunction function) throws SQLException;
PMetaData removeFunction(PName tenantId, String function, long functionTimeStamp) throws SQLException;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index 268bfc1..1cac37f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -96,6 +96,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE;
@@ -112,6 +113,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.schema.MetaDataSplitPolicy;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PNameFactory;
@@ -172,9 +174,15 @@ public interface QueryConstants {
public final static int MILLIS_IN_DAY = 1000 * 60 * 60 * 24;
public static final String EMPTY_COLUMN_NAME = "_0";
+ // For transactional tables, the value of our empty key value can no longer be empty
+ // since empty values are treated as column delete markers.
public static final byte[] EMPTY_COLUMN_BYTES = Bytes.toBytes(EMPTY_COLUMN_NAME);
public static final ImmutableBytesPtr EMPTY_COLUMN_BYTES_PTR = new ImmutableBytesPtr(
EMPTY_COLUMN_BYTES);
+ public final static String EMPTY_COLUMN_VALUE = "x";
+ public final static byte[] EMPTY_COLUMN_VALUE_BYTES = Bytes.toBytes(EMPTY_COLUMN_VALUE);
+ public static final ImmutableBytesPtr EMPTY_COLUMN_VALUE_BYTES_PTR = new ImmutableBytesPtr(
+ EMPTY_COLUMN_VALUE_BYTES);
public static final String DEFAULT_COLUMN_FAMILY = "0";
public static final byte[] DEFAULT_COLUMN_FAMILY_BYTES = Bytes.toBytes(DEFAULT_COLUMN_FAMILY);
@@ -251,13 +259,15 @@ public interface QueryConstants {
STORE_NULLS + " BOOLEAN," +
BASE_COLUMN_COUNT + " INTEGER," +
// Column metadata (will be null for table row)
- IS_ROW_TIMESTAMP + " BOOLEAN, " +
+ IS_ROW_TIMESTAMP + " BOOLEAN, " +
+ TRANSACTIONAL + " BOOLEAN," +
"CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ","
+ TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" +
HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
HColumnDescriptor.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n" +
// Install split policy to prevent a tenant's metadata from being split across regions.
- HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "'\n";
+ HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" +
+ PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
public static final String CREATE_STATS_TABLE_METADATA =
"CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_STATS_TABLE + "\"(\n" +
@@ -278,7 +288,8 @@ public interface QueryConstants {
HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_STAT_DATA_VERSIONS + ",\n" +
HColumnDescriptor.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n" +
// Install split policy to prevent a physical table's stats from being split across regions.
- HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "'\n";
+ HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" +
+ PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
public static final String CREATE_SEQUENCE_METADATA =
"CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + TYPE_SEQUENCE + "\"(\n" +
@@ -296,7 +307,8 @@ public interface QueryConstants {
LIMIT_REACHED_FLAG + " BOOLEAN \n" +
" CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + SEQUENCE_SCHEMA + "," + SEQUENCE_NAME + "))\n" +
HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
- HColumnDescriptor.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + "\n";
+ HColumnDescriptor.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n" +
+ PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
public static final String CREATE_FUNCTION_METADATA =
"CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_FUNCTION_TABLE + "\"(\n" +
@@ -320,6 +332,7 @@ public interface QueryConstants {
HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
HColumnDescriptor.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n"+
// Install split policy to prevent a tenant's metadata from being split across regions.
- HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "'\n";
+ HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "',\n" +
+ PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index df8cdc3..5f42ff8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -159,6 +159,7 @@ public interface QueryServices extends SQLCloseable {
public static final String DELAY_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.delay";
public static final String DEFAULT_KEEP_DELETED_CELLS_ATTRIB = "phoenix.table.default.keep.deleted.cells";
public static final String DEFAULT_STORE_NULLS_ATTRIB = "phoenix.table.default.store.nulls";
+ public static final String DEFAULT_TRANSACTIONAL_ATTRIB = "phoenix.transactions.default.enabled";
public static final String GLOBAL_METRICS_ENABLED = "phoenix.query.global.metrics.enabled";
// rpc queue configs
@@ -172,6 +173,8 @@ public interface QueryServices extends SQLCloseable {
public static final String RETURN_SEQUENCE_VALUES_ATTRIB = "phoenix.sequence.returnValues";
public static final String EXTRA_JDBC_ARGUMENTS_ATTRIB = "phoenix.jdbc.extra.arguments";
+ public static final String MAX_VERSIONS_TRANSACTIONAL_ATTRIB = "phoenix.transactions.maxVersions";
+
// queryserver configuration keys
public static final String QUERY_SERVER_SERIALIZATION_ATTRIB = "phoenix.queryserver.serialization";
public static final String QUERY_SERVER_META_FACTORY_ATTRIB = "phoenix.queryserver.metafactory.class";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 0434ac9..8c2f895 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -116,7 +116,7 @@ public class QueryServicesOptions {
public static final int DEFAULT_SCAN_CACHE_SIZE = 1000;
public static final int DEFAULT_MAX_INTRA_REGION_PARALLELIZATION = DEFAULT_MAX_QUERY_CONCURRENCY;
public static final int DEFAULT_DISTINCT_VALUE_COMPRESS_THRESHOLD = 1024 * 1024 * 1; // 1 Mb
- public static final int DEFAULT_INDEX_MUTATE_BATCH_SIZE_THRESHOLD = 5;
+ public static final int DEFAULT_INDEX_MUTATE_BATCH_SIZE_THRESHOLD = 3;
public static final long DEFAULT_MAX_SPOOL_TO_DISK_BYTES = 1024000000;
// Only the first chunked batches are fetched in parallel, so this default
// should be on the relatively bigger side of things. Bigger means more
@@ -190,7 +190,9 @@ public class QueryServicesOptions {
public static final boolean DEFAULT_STORE_NULLS = false;
// TODO Change this to true as part of PHOENIX-1543
+ // We'll also need this for transactions to work correctly
public static final boolean DEFAULT_AUTO_COMMIT = false;
+ public static final boolean DEFAULT_TRANSACTIONAL = false;
public static final boolean DEFAULT_IS_GLOBAL_METRICS_ENABLED = true;
private static final String DEFAULT_CLIENT_RPC_CONTROLLER_FACTORY = ClientRpcControllerFactory.class.getName();
@@ -202,6 +204,7 @@ public class QueryServicesOptions {
public static final boolean DEFAULT_ALLOW_USER_DEFINED_FUNCTIONS = false;
public static final boolean DEFAULT_REQUEST_LEVEL_METRICS_ENABLED = false;
public static final boolean DEFAULT_ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE = true;
+ public static final int DEFAULT_MAX_VERSIONS_TRANSACTIONAL = Integer.MAX_VALUE;
public static final boolean DEFAULT_RETURN_SEQUENCE_VALUES = false;
public static final String DEFAULT_EXTRA_JDBC_ARGUMENTS = "";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc9929b5/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java
index 5a0d63b..7fb90a1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java
@@ -162,8 +162,8 @@ public class DelegateTable implements PTable {
}
@Override
- public void getIndexMaintainers(ImmutableBytesWritable ptr, PhoenixConnection connection) {
- delegate.getIndexMaintainers(ptr, connection);
+ public boolean getIndexMaintainers(ImmutableBytesWritable ptr, PhoenixConnection connection) {
+ return delegate.getIndexMaintainers(ptr, connection);
}
@Override
@@ -238,6 +238,10 @@ public class DelegateTable implements PTable {
}
@Override
+ public boolean isTransactional() {
+ return delegate.isTransactional();
+ }
+
public int getBaseColumnCount() {
return delegate.getBaseColumnCount();
}
[24/52] [abbrv] phoenix git commit: PHOENIX-2462 Add tephra test
dependency to spark module
Posted by ma...@apache.org.
PHOENIX-2462 Add tephra test dependency to spark module
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/783ee1c7
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/783ee1c7
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/783ee1c7
Branch: refs/heads/calcite
Commit: 783ee1c7657b669c2bbb340e1f4e7ce402a26776
Parents: d60713e
Author: Josh Mahonin <jm...@interset.com>
Authored: Mon Nov 30 11:30:04 2015 -0500
Committer: Josh Mahonin <jm...@interset.com>
Committed: Mon Nov 30 11:31:00 2015 -0500
----------------------------------------------------------------------
phoenix-spark/pom.xml | 7 +++++++
.../it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala | 6 +++++-
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/783ee1c7/phoenix-spark/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-spark/pom.xml b/phoenix-spark/pom.xml
index 5edd5be..c0ad626 100644
--- a/phoenix-spark/pom.xml
+++ b/phoenix-spark/pom.xml
@@ -478,6 +478,13 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <type>test-jar</type>
+ <version>${tephra.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/783ee1c7/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala b/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
index 5ae17da..1a9be6b 100644
--- a/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
+++ b/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
@@ -40,7 +40,11 @@ import scala.collection.mutable.ListBuffer
// Helper object to access the protected abstract static methods hidden in BaseHBaseManagedTimeIT
object PhoenixSparkITHelper extends BaseHBaseManagedTimeIT {
def getTestClusterConfig = BaseHBaseManagedTimeIT.getTestClusterConfig
- def doSetup = BaseHBaseManagedTimeIT.doSetup()
+ def doSetup = {
+ // The @ClassRule doesn't seem to be getting picked up, force creation here before setup
+ BaseTest.tmpFolder.create()
+ BaseHBaseManagedTimeIT.doSetup()
+ }
def doTeardown = BaseHBaseManagedTimeIT.doTeardown()
def getUrl = BaseTest.getUrl
}
[25/52] [abbrv] phoenix git commit: PHOENIX-2461 Tweak shutdown of
ConnectionQueryService during testing
Posted by ma...@apache.org.
PHOENIX-2461 Tweak shutdown of ConnectionQueryService during testing
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3eb17d43
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3eb17d43
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3eb17d43
Branch: refs/heads/calcite
Commit: 3eb17d43fb996c034acb317395f5dd34e2a88c4c
Parents: 783ee1c
Author: James Taylor <jt...@salesforce.com>
Authored: Mon Nov 30 10:13:54 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Nov 30 10:14:06 2015 -0800
----------------------------------------------------------------------
.../apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3eb17d43/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
index 63dfaf9..d3b3f8a 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
@@ -67,7 +67,7 @@ public class ConnectionQueryServicesTestImpl extends ConnectionQueryServicesImpl
// Since ConnectionQueryServicesTestImpl is a singleton
// during testing, attempt to stop the Zookeeper client
// service when the driver is closed.
- if (zkClientService != null) zkClientService.stopAndWait();
+ // if (zkClientService != null) zkClientService.stopAndWait();
} finally {
super.close();
}
[16/52] [abbrv] phoenix git commit: PHOENIX-2444 Expansion of derived
table query works incorrectly with aggregate over order by
Posted by ma...@apache.org.
PHOENIX-2444 Expansion of derived table query works incorrectly with aggregate over order by
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7b2e1b14
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7b2e1b14
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7b2e1b14
Branch: refs/heads/calcite
Commit: 7b2e1b1493c1fe2bbcf8d2c8b5de6ba1447f7ec3
Parents: 1b5e7fd
Author: maryannxue <we...@intel.com>
Authored: Wed Nov 25 14:00:40 2015 -0500
Committer: maryannxue <we...@intel.com>
Committed: Wed Nov 25 14:00:40 2015 -0500
----------------------------------------------------------------------
.../apache/phoenix/end2end/DerivedTableIT.java | 25 ++++++++++++++++
.../phoenix/compile/SubselectRewriter.java | 31 +++++++++++---------
2 files changed, 42 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7b2e1b14/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java
index 7b21f37..95a7fed 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java
@@ -332,6 +332,22 @@ public class DerivedTableIT extends BaseClientManagedTimeIT {
rs = conn.createStatement().executeQuery("EXPLAIN " + query);
assertEquals(plans[1], QueryUtil.getExplainPlan(rs));
+
+ // (orderby) groupby
+ query = "SELECT a_string, count(*) FROM (SELECT * FROM aTable order by a_integer) AS t where a_byte != 8 group by a_string";
+ statement = conn.prepareStatement(query);
+ rs = statement.executeQuery();
+ assertTrue (rs.next());
+ assertEquals(A_VALUE,rs.getString(1));
+ assertEquals(4,rs.getInt(2));
+ assertTrue (rs.next());
+ assertEquals(B_VALUE,rs.getString(1));
+ assertEquals(3,rs.getInt(2));
+ assertTrue (rs.next());
+ assertEquals(C_VALUE,rs.getString(1));
+ assertEquals(1,rs.getInt(2));
+
+ assertFalse(rs.next());
} finally {
conn.close();
}
@@ -678,6 +694,15 @@ public class DerivedTableIT extends BaseClientManagedTimeIT {
assertEquals(8,rs.getInt(1));
assertFalse(rs.next());
+
+ // count (orderby)
+ query = "SELECT count(a_byte) FROM (SELECT * FROM aTable order by a_integer) AS t where a_byte != 8";
+ statement = conn.prepareStatement(query);
+ rs = statement.executeQuery();
+ assertTrue (rs.next());
+ assertEquals(8,rs.getInt(1));
+
+ assertFalse(rs.next());
} finally {
conn.close();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7b2e1b14/phoenix-core/src/main/java/org/apache/phoenix/compile/SubselectRewriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/SubselectRewriter.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/SubselectRewriter.java
index 5a91a17..853d772 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/SubselectRewriter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/SubselectRewriter.java
@@ -127,6 +127,22 @@ public class SubselectRewriter extends ParseNodeRewriter {
}
}
+ if (select.isDistinct()) {
+ if (subselect.getLimit() != null || subselect.isAggregate() || subselect.isDistinct()) {
+ return select;
+ }
+ isDistinctRewrite = true;
+ orderByRewrite = null;
+ }
+
+ if (select.isAggregate()) {
+ if (subselect.getLimit() != null || subselect.isAggregate() || subselect.isDistinct()) {
+ return select;
+ }
+ isAggregateRewrite = true;
+ orderByRewrite = null;
+ }
+
List<ParseNode> groupBy = select.getGroupBy();
if (!groupBy.isEmpty()) {
if (subselect.getLimit() != null || subselect.isAggregate() || subselect.isDistinct()) {
@@ -139,6 +155,7 @@ public class SubselectRewriter extends ParseNodeRewriter {
if (select.getHaving() != null) {
havingRewrite = select.getHaving().accept(this);
}
+ orderByRewrite = null;
}
List<AliasedNode> selectNodes = select.getSelect();
@@ -190,20 +207,6 @@ public class SubselectRewriter extends ParseNodeRewriter {
hintRewrite = hintRewrite == null ? hint : HintNode.combine(hint, hintRewrite);
}
- if (select.isDistinct()) {
- if (subselect.getLimit() != null || subselect.isAggregate() || subselect.isDistinct()) {
- return select;
- }
- isDistinctRewrite = true;
- }
-
- if (select.isAggregate()) {
- if (subselect.getLimit() != null || subselect.isAggregate() || subselect.isDistinct()) {
- return select;
- }
- isAggregateRewrite = true;
- }
-
return NODE_FACTORY.select(subselect.getFrom(), hintRewrite, isDistinctRewrite, selectNodesRewrite, whereRewrite, groupByRewrite,
havingRewrite, orderByRewrite, limitRewrite, select.getBindCount(), isAggregateRewrite, select.hasSequence(), select.getSelects(), select.getUdfParseNodes());
}
[35/52] [abbrv] phoenix git commit: PHOENIX-2468 Add dynamic column
to track asynchronous index creation date
Posted by ma...@apache.org.
PHOENIX-2468 Add dynamic column to track asynchronous index creation date
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/30ef0cc5
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/30ef0cc5
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/30ef0cc5
Branch: refs/heads/calcite
Commit: 30ef0cc5d3a86002af45b5fb0a7345c31225cc9b
Parents: 0701e06
Author: James Taylor <jt...@salesforce.com>
Authored: Wed Dec 2 15:08:10 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Dec 2 15:08:29 2015 -0800
----------------------------------------------------------------------
.../phoenix/end2end/index/IndexMetadataIT.java | 40 ++++++++++++++++++++
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 2 +
.../apache/phoenix/schema/MetaDataClient.java | 30 ++++++++++++---
3 files changed, 67 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/30ef0cc5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
index e72ad0c..1af15a1 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
@@ -28,6 +28,7 @@ import static org.junit.Assert.fail;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
+import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
@@ -39,12 +40,14 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.AmbiguousColumnException;
import org.apache.phoenix.schema.PIndexState;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.TableNotFoundException;
+import org.apache.phoenix.schema.types.PDate;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.StringUtil;
@@ -470,4 +473,41 @@ public class IndexMetadataIT extends BaseHBaseManagedTimeIT {
conn.close();
}
}
+
+ @Test
+ public void testAsyncCreatedDate() throws Exception {
+ Date d0 = new Date(System.currentTimeMillis());
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ String ddl = "create table test_table (k varchar primary key, v1 varchar, v2 varchar, v3 varchar)";
+ PreparedStatement stmt = conn.prepareStatement(ddl);
+ stmt.execute();
+
+ ddl = "CREATE INDEX IDX1 ON test_table (v1) ASYNC";
+ stmt = conn.prepareStatement(ddl);
+ stmt.execute();
+ ddl = "CREATE INDEX IDX2 ON test_table (v2) ASYNC";
+ stmt = conn.prepareStatement(ddl);
+ stmt.execute();
+ ddl = "CREATE INDEX IDX3 ON test_table (v3)";
+ stmt = conn.prepareStatement(ddl);
+ stmt.execute();
+
+ ResultSet rs = conn.createStatement().executeQuery(
+ "select table_name, " + PhoenixDatabaseMetaData.ASYNC_CREATED_DATE + " " +
+ "from system.catalog (" + PhoenixDatabaseMetaData.ASYNC_CREATED_DATE + " " + PDate.INSTANCE.getSqlTypeName() + ") " +
+ "where " + PhoenixDatabaseMetaData.ASYNC_CREATED_DATE + " is not null " +
+ "order by " + PhoenixDatabaseMetaData.ASYNC_CREATED_DATE
+ );
+ assertTrue(rs.next());
+ assertEquals("IDX1", rs.getString(1));
+ Date d1 = rs.getDate(2);
+ assertTrue(d1.after(d0));
+ assertTrue(rs.next());
+ assertEquals("IDX2", rs.getString(1));
+ Date d2 = rs.getDate(2);
+ assertTrue(d2.after(d1));
+ assertFalse(rs.next());
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/30ef0cc5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index d6d1230..940ca52 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -304,6 +304,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData, org.apache.pho
public static final String TRANSACTIONAL = "TRANSACTIONAL";
public static final byte[] TRANSACTIONAL_BYTES = Bytes.toBytes(TRANSACTIONAL);
+ public static final String ASYNC_CREATED_DATE = "ASYNC_CREATED_DATE";
+
private final PhoenixConnection connection;
private final ResultSet emptyResultSet;
public static final int MAX_LOCAL_SI_VERSION_DISALLOW = VersionUtil.encodeVersion("0", "98", "8");
http://git-wip-us.apache.org/repos/asf/phoenix/blob/30ef0cc5/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index cabfd04..ccfb1cb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -24,6 +24,7 @@ import static org.apache.hadoop.hbase.HColumnDescriptor.TTL;
import static org.apache.phoenix.exception.SQLExceptionCode.INSUFFICIENT_MULTI_TENANT_COLUMNS;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ASYNC_CREATED_DATE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BASE_COLUMN_COUNT;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLASS_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT;
@@ -80,15 +81,14 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB;
-import static org.apache.phoenix.query.QueryServices.TRANSACTIONS_ENABLED;
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_DROP_METADATA;
-import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED;
import static org.apache.phoenix.schema.PTable.ViewType.MAPPED;
import static org.apache.phoenix.schema.PTableType.TABLE;
import static org.apache.phoenix.schema.PTableType.VIEW;
import java.io.IOException;
import java.sql.Connection;
+import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
@@ -182,6 +182,7 @@ import org.apache.phoenix.schema.PTable.LinkType;
import org.apache.phoenix.schema.PTable.ViewType;
import org.apache.phoenix.schema.stats.PTableStats;
import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PDate;
import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.schema.types.PTimestamp;
@@ -216,6 +217,13 @@ public class MetaDataClient {
private static final Logger logger = LoggerFactory.getLogger(MetaDataClient.class);
private static final ParseNodeFactory FACTORY = new ParseNodeFactory();
+ private static final String SET_ASYNC_CREATED_DATE =
+ "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " +
+ TENANT_ID + "," +
+ TABLE_SCHEM + "," +
+ TABLE_NAME + "," +
+ ASYNC_CREATED_DATE + " " + PDate.INSTANCE.getSqlTypeName() +
+ ") VALUES (?, ?, ?, ?)";
private static final String CREATE_TABLE =
"UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " +
TENANT_ID + "," +
@@ -827,7 +835,7 @@ public class MetaDataClient {
}
public MutationState createTable(CreateTableStatement statement, byte[][] splits, PTable parent, String viewStatement, ViewType viewType, byte[][] viewColumnConstants, BitSet isViewColumnReferenced) throws SQLException {
- PTable table = createTableInternal(statement, splits, parent, viewStatement, viewType, viewColumnConstants, isViewColumnReferenced, null, null);
+ PTable table = createTableInternal(statement, splits, parent, viewStatement, viewType, viewColumnConstants, isViewColumnReferenced, null, null, null);
if (table == null || table.getType() == PTableType.VIEW || table.isTransactional()) {
return new MutationState(0,connection);
}
@@ -1172,6 +1180,10 @@ public class MetaDataClient {
try {
ColumnResolver resolver = FromCompiler.getResolver(statement, connection, statement.getUdfParseNodes());
tableRef = resolver.getTables().get(0);
+ Date asyncCreatedDate = null;
+ if (statement.isAsync()) {
+ asyncCreatedDate = new Date(tableRef.getTimeStamp());
+ }
PTable dataTable = tableRef.getTable();
boolean isTenantConnection = connection.getTenantId() != null;
if (isTenantConnection) {
@@ -1346,7 +1358,7 @@ public class MetaDataClient {
}
PrimaryKeyConstraint pk = FACTORY.primaryKey(null, allPkColumns);
CreateTableStatement tableStatement = FACTORY.createTable(indexTableName, statement.getProps(), columnDefs, pk, statement.getSplitNodes(), PTableType.INDEX, statement.ifNotExists(), null, null, statement.getBindCount());
- table = createTableInternal(tableStatement, splits, dataTable, null, null, null, null, indexId, statement.getIndexType());
+ table = createTableInternal(tableStatement, splits, dataTable, null, null, null, null, indexId, statement.getIndexType(), asyncCreatedDate);
break;
} catch (ConcurrentTableMutationException e) { // Can happen if parent data table changes while above is in progress
if (retry) {
@@ -1526,7 +1538,7 @@ public class MetaDataClient {
return false;
}
- private PTable createTableInternal(CreateTableStatement statement, byte[][] splits, final PTable parent, String viewStatement, ViewType viewType, final byte[][] viewColumnConstants, final BitSet isViewColumnReferenced, Short indexId, IndexType indexType) throws SQLException {
+ private PTable createTableInternal(CreateTableStatement statement, byte[][] splits, final PTable parent, String viewStatement, ViewType viewType, final byte[][] viewColumnConstants, final BitSet isViewColumnReferenced, Short indexId, IndexType indexType, Date asyncCreatedDate) throws SQLException {
final PTableType tableType = statement.getTableType();
boolean wasAutoCommit = connection.getAutoCommit();
connection.rollback();
@@ -2099,6 +2111,14 @@ public class MetaDataClient {
tableUpsert.setBoolean(21, transactional);
tableUpsert.execute();
+ if (asyncCreatedDate != null) {
+ PreparedStatement setAsync = connection.prepareStatement(SET_ASYNC_CREATED_DATE);
+ setAsync.setString(1, tenantIdStr);
+ setAsync.setString(2, schemaName);
+ setAsync.setString(3, tableName);
+ setAsync.setDate(4, asyncCreatedDate);
+ setAsync.execute();
+ }
tableMetaData.addAll(connection.getMutationState().toMutations(timestamp).next().getSecond());
connection.rollback();