You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by sa...@apache.org on 2016/01/07 00:17:14 UTC
[01/50] [abbrv] phoenix git commit: PHOENIX-2484 Exclude logback impl
from Tephra
Repository: phoenix
Updated Branches:
refs/heads/4.x-HBase-1.0 3e798008d -> 5779f263c
PHOENIX-2484 Exclude logback impl from Tephra
Exclude the logback dependency that is pulled in by Tephra,
allowing Tephra's SLF4J logging to be delegated into log4j,
and having everything configured via a single log4j.properties
config.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/f1579067
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/f1579067
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/f1579067
Branch: refs/heads/4.x-HBase-1.0
Commit: f15790679f8f9cb95b28562266cc198f0e4309fc
Parents: deffe07
Author: Gabriel Reid <ga...@ngdata.com>
Authored: Fri Dec 11 13:21:41 2015 +0100
Committer: Gabriel Reid <gr...@apache.org>
Committed: Sun Dec 13 06:24:48 2015 +0100
----------------------------------------------------------------------
phoenix-core/pom.xml | 4 ----
phoenix-flume/pom.xml | 1 -
phoenix-pherf/pom.xml | 9 ++++-----
phoenix-pig/pom.xml | 1 -
phoenix-server/pom.xml | 1 -
phoenix-spark/pom.xml | 1 -
pom.xml | 44 ++++++++++++++++++++++++++++++++++++++++++++
7 files changed, 48 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f1579067/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index f504a8f..73b4795 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -226,24 +226,20 @@
<dependency>
<groupId>co.cask.tephra</groupId>
<artifactId>tephra-api</artifactId>
- <version>${tephra.version}</version>
</dependency>
<dependency>
<groupId>co.cask.tephra</groupId>
<artifactId>tephra-core</artifactId>
- <version>${tephra.version}</version>
</dependency>
<dependency>
<groupId>co.cask.tephra</groupId>
<artifactId>tephra-core</artifactId>
<type>test-jar</type>
- <version>${tephra.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>co.cask.tephra</groupId>
<artifactId>tephra-hbase-compat-1.0</artifactId>
- <version>${tephra.version}</version>
</dependency>
<!-- Make sure we have all the antlr dependencies -->
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f1579067/phoenix-flume/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-flume/pom.xml b/phoenix-flume/pom.xml
index b2b131e..35db13b 100644
--- a/phoenix-flume/pom.xml
+++ b/phoenix-flume/pom.xml
@@ -177,7 +177,6 @@
<groupId>co.cask.tephra</groupId>
<artifactId>tephra-core</artifactId>
<type>test-jar</type>
- <version>${tephra.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f1579067/phoenix-pherf/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-pherf/pom.xml b/phoenix-pherf/pom.xml
index de3a2e3..8dc3977 100644
--- a/phoenix-pherf/pom.xml
+++ b/phoenix-pherf/pom.xml
@@ -100,11 +100,10 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>co.cask.tephra</groupId>
- <artifactId>tephra-core</artifactId>
- <type>test-jar</type>
- <version>${tephra.version}</version>
- <scope>test</scope>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f1579067/phoenix-pig/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-pig/pom.xml b/phoenix-pig/pom.xml
index 8d6cb7e..29ceb86 100644
--- a/phoenix-pig/pom.xml
+++ b/phoenix-pig/pom.xml
@@ -151,7 +151,6 @@
<groupId>co.cask.tephra</groupId>
<artifactId>tephra-core</artifactId>
<type>test-jar</type>
- <version>${tephra.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f1579067/phoenix-server/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-server/pom.xml b/phoenix-server/pom.xml
index b128d6a..c9b7b17 100644
--- a/phoenix-server/pom.xml
+++ b/phoenix-server/pom.xml
@@ -110,7 +110,6 @@
<groupId>co.cask.tephra</groupId>
<artifactId>tephra-core</artifactId>
<type>test-jar</type>
- <version>${tephra.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f1579067/phoenix-spark/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-spark/pom.xml b/phoenix-spark/pom.xml
index a0f6abd..af37b1e 100644
--- a/phoenix-spark/pom.xml
+++ b/phoenix-spark/pom.xml
@@ -475,7 +475,6 @@
<groupId>co.cask.tephra</groupId>
<artifactId>tephra-core</artifactId>
<type>test-jar</type>
- <version>${tephra.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f1579067/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 97f9f90..0b65a66 100644
--- a/pom.xml
+++ b/pom.xml
@@ -590,6 +590,50 @@
<version>${calcite.version}</version>
</dependency>
+ <!-- Transaction dependencies -->
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-api</artifactId>
+ <version>${tephra.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <version>${tephra.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-core</artifactId>
+ <type>test-jar</type>
+ <version>${tephra.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>co.cask.tephra</groupId>
+ <artifactId>tephra-hbase-compat-1.0</artifactId>
+ <version>${tephra.version}</version>
+ </dependency>
+
<!-- Make sure we have all the antlr dependencies -->
<dependency>
<groupId>org.antlr</groupId>
[12/50] [abbrv] phoenix git commit: PHOENIX-2533 Phoenix compilation
fails as ZKUtil.applyClusterKeyToConf method is removed (Ravi Kishore Valeti)
Posted by sa...@apache.org.
PHOENIX-2533 Phoenix compilation fails as ZKUtil.applyClusterKeyToConf method is removed (Ravi Kishore Valeti)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a2b40de6
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a2b40de6
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a2b40de6
Branch: refs/heads/4.x-HBase-1.0
Commit: a2b40de63b8ca8fe33a7b7d09d7535fd4689b895
Parents: 356b73f
Author: James Taylor <jt...@salesforce.com>
Authored: Wed Dec 16 09:22:25 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Dec 16 09:24:51 2015 -0800
----------------------------------------------------------------------
.../mapreduce/index/DirectHTableWriter.java | 38 ++------------------
1 file changed, 2 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a2b40de6/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/DirectHTableWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/DirectHTableWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/DirectHTableWriter.java
index d18fde9..59b26b2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/DirectHTableWriter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/DirectHTableWriter.java
@@ -22,10 +22,9 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Mutation;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,25 +38,6 @@ public class DirectHTableWriter {
private HTable table;
- /** Job parameter that specifies the output table. */
- public static final String OUTPUT_TABLE = "hbase.mapred.outputtable";
-
- /**
- * Optional job parameter to specify a peer cluster. Used specifying remote cluster when copying
- * between hbase clusters (the source is picked up from <code>hbase-site.xml</code>).
- * @see TableMapReduceUtil#initTableReducerJob(String, Class, org.apache.hadoop.mapreduce.Job,
- * Class, String, String, String)
- */
- public static final String QUORUM_ADDRESS = "hbase.mapred.output.quorum";
-
- /** Optional job parameter to specify peer cluster's ZK client port */
- public static final String QUORUM_PORT = "hbase.mapred.output.quorum.port";
-
- /** Optional specification of the rs class name of the peer cluster */
- public static final String REGION_SERVER_CLASS = "hbase.mapred.output.rs.class";
- /** Optional specification of the rs impl name of the peer cluster */
- public static final String REGION_SERVER_IMPL = "hbase.mapred.output.rs.impl";
-
public DirectHTableWriter(Configuration otherConf) {
setConf(otherConf);
}
@@ -65,26 +45,12 @@ public class DirectHTableWriter {
protected void setConf(Configuration otherConf) {
this.conf = HBaseConfiguration.create(otherConf);
- String tableName = this.conf.get(OUTPUT_TABLE);
+ String tableName = this.conf.get(TableOutputFormat.OUTPUT_TABLE);
if (tableName == null || tableName.length() <= 0) {
throw new IllegalArgumentException("Must specify table name");
}
- String address = this.conf.get(QUORUM_ADDRESS);
- int zkClientPort = this.conf.getInt(QUORUM_PORT, 0);
- String serverClass = this.conf.get(REGION_SERVER_CLASS);
- String serverImpl = this.conf.get(REGION_SERVER_IMPL);
-
try {
- if (address != null) {
- ZKUtil.applyClusterKeyToConf(this.conf, address);
- }
- if (serverClass != null) {
- this.conf.set(HConstants.REGION_SERVER_IMPL, serverImpl);
- }
- if (zkClientPort != 0) {
- this.conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
- }
this.table = new HTable(this.conf, tableName);
this.table.setAutoFlush(false, true);
LOG.info("Created table instance for " + tableName);
[29/50] [abbrv] phoenix git commit: PHOENIX-2546 Filters should take
into account that multiple versions may be scanned
Posted by sa...@apache.org.
PHOENIX-2546 Filters should take into account that multiple versions may be scanned
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/267715aa
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/267715aa
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/267715aa
Branch: refs/heads/4.x-HBase-1.0
Commit: 267715aae9022b21c664e3721c2fbf1e6bf5fde6
Parents: ca10171
Author: James Taylor <jt...@salesforce.com>
Authored: Tue Dec 29 23:00:27 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Dec 30 18:14:25 2015 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/tx/TransactionIT.java | 147 +++++++++++++++++++
.../phoenix/filter/ColumnProjectionFilter.java | 3 +-
.../MultiCFCQKeyValueComparisonFilter.java | 1 -
.../filter/MultiCQKeyValueComparisonFilter.java | 2 -
.../filter/MultiKeyValueComparisonFilter.java | 69 ++++-----
.../phoenix/filter/RowKeyComparisonFilter.java | 11 +-
.../SingleCFCQKeyValueComparisonFilter.java | 3 -
.../SingleCQKeyValueComparisonFilter.java | 3 -
.../filter/SingleKeyValueComparisonFilter.java | 19 +--
.../apache/phoenix/filter/SkipScanFilter.java | 6 +-
.../schema/tuple/SingleKeyValueTuple.java | 46 +++---
.../phoenix/filter/SkipScanFilterTest.java | 2 +-
12 files changed, 217 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/267715aa/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
index e83467a..63a5d6e 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
@@ -62,6 +62,7 @@ import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.TestUtil;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.Lists;
@@ -350,6 +351,7 @@ public class TransactionIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
}
+ @Ignore
@Test
public void testNonTxToTxTableFailure() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
@@ -725,4 +727,149 @@ public class TransactionIT extends BaseHBaseManagedTimeIT {
Result result = htable.get(new Get(Bytes.toBytes("j")));
assertTrue(result.isEmpty());
}
+
+ @Test
+ public void testCheckpointAndRollback() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ String fullTableName = "T";
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) TRANSACTIONAL=true");
+ stmt.executeUpdate("upsert into " + fullTableName + " values('x', 'a', 'a')");
+ conn.commit();
+
+ stmt.executeUpdate("upsert into " + fullTableName + "(k,v1) SELECT k,v1||'a' FROM " + fullTableName);
+ ResultSet rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("aa", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ stmt.executeUpdate("upsert into " + fullTableName + "(k,v1) SELECT k,v1||'a' FROM " + fullTableName);
+
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("aaa", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ conn.rollback();
+
+ //assert original row exists in fullTableName1
+ rs = stmt.executeQuery("select k, v1, v2 from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals("a", rs.getString(2));
+ assertEquals("a", rs.getString(3));
+ assertFalse(rs.next());
+
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Ignore("Add back once TEPHRA-162 gets fixed")
+ @Test
+ public void testInflightUpdateNotSeen() throws Exception {
+ String selectSQL = "SELECT * FROM " + FULL_TABLE_NAME;
+ try (Connection conn1 = DriverManager.getConnection(getUrl());
+ Connection conn2 = DriverManager.getConnection(getUrl())) {
+ conn1.setAutoCommit(false);
+ conn2.setAutoCommit(true);
+ ResultSet rs = conn1.createStatement().executeQuery(selectSQL);
+ assertFalse(rs.next());
+
+ String upsert = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn1.prepareStatement(upsert);
+ // upsert two rows
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.execute();
+ conn1.commit();
+
+ TestUtil.setRowKeyColumns(stmt, 2);
+ stmt.execute();
+
+ rs = conn1.createStatement().executeQuery("SELECT count(*) FROM " + FULL_TABLE_NAME + " WHERE int_col1 IS NULL");
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+
+ upsert = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk, int_col1) VALUES(?, ?, ?, ?, ?, ?, 1)";
+ stmt = conn1.prepareStatement(upsert);
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.execute();
+
+ rs = conn1.createStatement().executeQuery("SELECT int_col1 FROM " + FULL_TABLE_NAME + " WHERE int_col1 = 1");
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertFalse(rs.next());
+
+ rs = conn2.createStatement().executeQuery("SELECT count(*) FROM " + FULL_TABLE_NAME + " WHERE int_col1 = 1");
+ assertTrue(rs.next());
+ assertEquals(0, rs.getInt(1));
+ rs = conn2.createStatement().executeQuery("SELECT * FROM " + FULL_TABLE_NAME + " WHERE int_col1 = 1");
+ assertFalse(rs.next());
+
+ conn1.commit();
+
+ rs = conn2.createStatement().executeQuery("SELECT count(*) FROM " + FULL_TABLE_NAME + " WHERE int_col1 = 1");
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ rs = conn2.createStatement().executeQuery("SELECT * FROM " + FULL_TABLE_NAME + " WHERE int_col1 = 1");
+ assertTrue(rs.next());
+ assertFalse(rs.next());
+ }
+ }
+
+ @Ignore("Add back once TEPHRA-162 gets fixed")
+ @Test
+ public void testInflightDeleteNotSeen() throws Exception {
+ String selectSQL = "SELECT * FROM " + FULL_TABLE_NAME;
+ try (Connection conn1 = DriverManager.getConnection(getUrl());
+ Connection conn2 = DriverManager.getConnection(getUrl())) {
+ conn1.setAutoCommit(false);
+ conn2.setAutoCommit(true);
+ ResultSet rs = conn1.createStatement().executeQuery(selectSQL);
+ assertFalse(rs.next());
+
+ String upsert = "UPSERT INTO " + FULL_TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn1.prepareStatement(upsert);
+ // upsert two rows
+ TestUtil.setRowKeyColumns(stmt, 1);
+ stmt.execute();
+ TestUtil.setRowKeyColumns(stmt, 2);
+ stmt.execute();
+
+ conn1.commit();
+
+ rs = conn1.createStatement().executeQuery("SELECT count(*) FROM " + FULL_TABLE_NAME);
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+
+ String delete = "DELETE FROM " + FULL_TABLE_NAME + " WHERE varchar_pk = 'varchar1'";
+ stmt = conn1.prepareStatement(delete);
+ int count = stmt.executeUpdate();
+ assertEquals(1,count);
+
+ rs = conn1.createStatement().executeQuery("SELECT count(*) FROM " + FULL_TABLE_NAME);
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertFalse(rs.next());
+
+ rs = conn2.createStatement().executeQuery("SELECT count(*) FROM " + FULL_TABLE_NAME);
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+ assertFalse(rs.next());
+
+ conn1.commit();
+
+ rs = conn2.createStatement().executeQuery("SELECT count(*) FROM " + FULL_TABLE_NAME);
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertFalse(rs.next());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/267715aa/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
index a238e8e..cf9f7ab 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
-import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.filter.FilterBase;
import org.apache.hadoop.hbase.util.Bytes;
@@ -175,6 +174,6 @@ public class ColumnProjectionFilter extends FilterBase implements Writable {
@Override
public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE;
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/267715aa/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiCFCQKeyValueComparisonFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiCFCQKeyValueComparisonFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiCFCQKeyValueComparisonFilter.java
index 9147f1a..3bd1fd7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiCFCQKeyValueComparisonFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiCFCQKeyValueComparisonFilter.java
@@ -31,7 +31,6 @@ import org.apache.phoenix.expression.Expression;
* are references to multiple column qualifiers over multiple column families.
* Also there same qualifier names in different families.
*
- * @since 0.1
*/
public class MultiCFCQKeyValueComparisonFilter extends MultiKeyValueComparisonFilter {
private final ImmutablePairBytesPtr ptr = new ImmutablePairBytesPtr();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/267715aa/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiCQKeyValueComparisonFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiCQKeyValueComparisonFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiCQKeyValueComparisonFilter.java
index 5fa5035..91e4392 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiCQKeyValueComparisonFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiCQKeyValueComparisonFilter.java
@@ -29,8 +29,6 @@ import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
* Filter that evaluates WHERE clause expression, used in the case where there
* are references to multiple unique column qualifiers over one or more column families.
*
- *
- * @since 0.1
*/
public class MultiCQKeyValueComparisonFilter extends MultiKeyValueComparisonFilter {
private ImmutableBytesPtr ptr = new ImmutableBytesPtr();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/267715aa/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java
index 1cb2255..569faa5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java
@@ -24,8 +24,6 @@ import java.util.Map;
import java.util.TreeSet;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.expression.Expression;
@@ -41,8 +39,6 @@ import org.apache.phoenix.schema.tuple.BaseTuple;
* but for general expression evaluation in the case where multiple KeyValue
* columns are referenced in the expression.
*
- *
- * @since 0.1
*/
public abstract class MultiKeyValueComparisonFilter extends BooleanExpressionFilter {
private static final byte[] UNITIALIZED_KEY_BUFFER = new byte[0];
@@ -59,14 +55,14 @@ public abstract class MultiKeyValueComparisonFilter extends BooleanExpressionFil
init();
}
- private static final class KeyValueRef {
- public KeyValue keyValue;
+ private static final class CellRef {
+ public Cell cell;
@Override
public String toString() {
- if(keyValue != null) {
- return keyValue.toString() + " value = " + Bytes.toStringBinary(
- keyValue.getValueArray(), keyValue.getValueOffset(), keyValue.getValueLength());
+ if(cell != null) {
+ return cell.toString() + " value = " + Bytes.toStringBinary(
+ cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
} else {
return super.toString();
}
@@ -79,13 +75,13 @@ public abstract class MultiKeyValueComparisonFilter extends BooleanExpressionFil
private final class IncrementalResultTuple extends BaseTuple {
private int refCount;
private final ImmutableBytesWritable keyPtr = new ImmutableBytesWritable(UNITIALIZED_KEY_BUFFER);
- private final Map<Object,KeyValueRef> foundColumns = new HashMap<Object,KeyValueRef>(5);
+ private final Map<Object,CellRef> foundColumns = new HashMap<Object,CellRef>(5);
public void reset() {
refCount = 0;
keyPtr.set(UNITIALIZED_KEY_BUFFER);
- for (KeyValueRef ref : foundColumns.values()) {
- ref.keyValue = null;
+ for (CellRef ref : foundColumns.values()) {
+ ref.cell = null;
}
}
@@ -98,39 +94,39 @@ public abstract class MultiKeyValueComparisonFilter extends BooleanExpressionFil
refCount = foundColumns.size();
}
- public ReturnCode resolveColumn(KeyValue value) {
+ public ReturnCode resolveColumn(Cell value) {
// Always set key, in case we never find a key value column of interest,
// and our expression uses row key columns.
setKey(value);
Object ptr = setColumnKey(value.getFamilyArray(), value.getFamilyOffset(), value.getFamilyLength(),
value.getQualifierArray(), value.getQualifierOffset(), value.getQualifierLength());
- KeyValueRef ref = foundColumns.get(ptr);
+ CellRef ref = foundColumns.get(ptr);
if (ref == null) {
- // Return INCLUDE here. Although this filter doesn't need this KV
+ // Return INCLUDE_AND_NEXT_COL here. Although this filter doesn't need this KV
// it should still be projected into the Result
- return ReturnCode.INCLUDE;
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
}
// Since we only look at the latest key value for a given column,
// we are not interested in older versions
// TODO: test with older versions to confirm this doesn't get tripped
// This shouldn't be necessary, because a scan only looks at the latest
// version
- if (ref.keyValue != null) {
+ if (ref.cell != null) {
// Can't do NEXT_ROW, because then we don't match the other columns
// SKIP, INCLUDE, and NEXT_COL seem to all act the same
return ReturnCode.NEXT_COL;
}
- ref.keyValue = value;
+ ref.cell = value;
refCount++;
return null;
}
public void addColumn(byte[] cf, byte[] cq) {
Object ptr = MultiKeyValueComparisonFilter.this.newColumnKey(cf, 0, cf.length, cq, 0, cq.length);
- foundColumns.put(ptr, new KeyValueRef());
+ foundColumns.put(ptr, new CellRef());
}
- public void setKey(KeyValue value) {
+ public void setKey(Cell value) {
keyPtr.set(value.getRowArray(), value.getRowOffset(), value.getRowLength());
}
@@ -140,10 +136,10 @@ public abstract class MultiKeyValueComparisonFilter extends BooleanExpressionFil
}
@Override
- public KeyValue getValue(byte[] cf, byte[] cq) {
+ public Cell getValue(byte[] cf, byte[] cq) {
Object ptr = setColumnKey(cf, 0, cf.length, cq, 0, cq.length);
- KeyValueRef ref = foundColumns.get(ptr);
- return ref == null ? null : ref.keyValue;
+ CellRef ref = foundColumns.get(ptr);
+ return ref == null ? null : ref.cell;
}
@Override
@@ -157,15 +153,15 @@ public abstract class MultiKeyValueComparisonFilter extends BooleanExpressionFil
}
@Override
- public KeyValue getValue(int index) {
+ public Cell getValue(int index) {
// This won't perform very well, but it's not
// currently used anyway
- for (KeyValueRef ref : foundColumns.values()) {
- if (ref.keyValue == null) {
+ for (CellRef ref : foundColumns.values()) {
+ if (ref.cell == null) {
continue;
}
if (index == 0) {
- return ref.keyValue;
+ return ref.cell;
}
index--;
}
@@ -175,10 +171,10 @@ public abstract class MultiKeyValueComparisonFilter extends BooleanExpressionFil
@Override
public boolean getValue(byte[] family, byte[] qualifier,
ImmutableBytesWritable ptr) {
- KeyValue kv = getValue(family, qualifier);
- if (kv == null)
+ Cell cell = getValue(family, qualifier);
+ if (cell == null)
return false;
- ptr.set(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength());
+ ptr.set(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
return true;
}
}
@@ -197,17 +193,17 @@ public abstract class MultiKeyValueComparisonFilter extends BooleanExpressionFil
}
@Override
- public ReturnCode filterKeyValue(Cell keyValue) {
+ public ReturnCode filterKeyValue(Cell cell) {
if (Boolean.TRUE.equals(this.matchedColumn)) {
// We already found and matched the single column, all keys now pass
- return ReturnCode.INCLUDE;
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
}
if (Boolean.FALSE.equals(this.matchedColumn)) {
// We found all the columns, but did not match the expression, so skip to next row
return ReturnCode.NEXT_ROW;
}
// This is a key value we're not interested in (TODO: why INCLUDE here instead of NEXT_COL?)
- ReturnCode code = inputTuple.resolveColumn(KeyValueUtil.ensureKeyValue(keyValue));
+ ReturnCode code = inputTuple.resolveColumn(cell);
if (code != null) {
return code;
}
@@ -220,10 +216,10 @@ public abstract class MultiKeyValueComparisonFilter extends BooleanExpressionFil
if (inputTuple.isImmutable()) {
this.matchedColumn = Boolean.FALSE;
} else {
- return ReturnCode.INCLUDE;
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
}
}
- return this.matchedColumn ? ReturnCode.INCLUDE : ReturnCode.NEXT_ROW;
+ return this.matchedColumn ? ReturnCode.INCLUDE_AND_NEXT_COL : ReturnCode.NEXT_ROW;
}
@Override
@@ -243,8 +239,7 @@ public abstract class MultiKeyValueComparisonFilter extends BooleanExpressionFil
super.reset();
}
- @SuppressWarnings("all")
- // suppressing missing @Override since this doesn't exist for HBase 0.94.4
+ @Override
public boolean isFamilyEssential(byte[] name) {
// Only the column families involved in the expression are essential.
// The others are for columns projected in the select expression.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/267715aa/phoenix-core/src/main/java/org/apache/phoenix/filter/RowKeyComparisonFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/RowKeyComparisonFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/RowKeyComparisonFilter.java
index b7de7ac..2eb69bd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/RowKeyComparisonFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/RowKeyComparisonFilter.java
@@ -22,7 +22,6 @@ import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
@@ -38,8 +37,6 @@ import org.slf4j.LoggerFactory;
*
* Filter for use when expressions only reference row key columns
*
- *
- * @since 0.1
*/
public class RowKeyComparisonFilter extends BooleanExpressionFilter {
private static final Logger logger = LoggerFactory.getLogger(RowKeyComparisonFilter.class);
@@ -79,7 +76,7 @@ public class RowKeyComparisonFilter extends BooleanExpressionFilter {
}
evaluate = false;
}
- return keepRow ? ReturnCode.INCLUDE : ReturnCode.NEXT_ROW;
+ return keepRow ? ReturnCode.INCLUDE_AND_NEXT_COL : ReturnCode.NEXT_ROW;
}
private final class RowKeyTuple extends BaseTuple {
@@ -99,7 +96,7 @@ public class RowKeyComparisonFilter extends BooleanExpressionFilter {
}
@Override
- public KeyValue getValue(byte[] cf, byte[] cq) {
+ public Cell getValue(byte[] cf, byte[] cq) {
return null;
}
@@ -119,7 +116,7 @@ public class RowKeyComparisonFilter extends BooleanExpressionFilter {
}
@Override
- public KeyValue getValue(int index) {
+ public Cell getValue(int index) {
throw new IndexOutOfBoundsException(Integer.toString(index));
}
@@ -135,7 +132,7 @@ public class RowKeyComparisonFilter extends BooleanExpressionFilter {
return !this.keepRow;
}
- @SuppressWarnings("all") // suppressing missing @Override since this doesn't exist for HBase 0.94.4
+ @Override
public boolean isFamilyEssential(byte[] name) {
// We only need our "guaranteed to have a key value" column family,
// which we pass in and serialize through. In the case of a VIEW where
http://git-wip-us.apache.org/repos/asf/phoenix/blob/267715aa/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleCFCQKeyValueComparisonFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleCFCQKeyValueComparisonFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleCFCQKeyValueComparisonFilter.java
index 963fe59..c63673c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleCFCQKeyValueComparisonFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleCFCQKeyValueComparisonFilter.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Writables;
-
import org.apache.phoenix.expression.Expression;
@@ -32,8 +31,6 @@ import org.apache.phoenix.expression.Expression;
* column qualifier parts of the key value to disambiguate with another similarly
* named column qualifier in a different column family.
*
- *
- * @since 0.1
*/
public class SingleCFCQKeyValueComparisonFilter extends SingleKeyValueComparisonFilter {
public SingleCFCQKeyValueComparisonFilter() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/267715aa/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleCQKeyValueComparisonFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleCQKeyValueComparisonFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleCQKeyValueComparisonFilter.java
index 240c8a6..0d904bc 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleCQKeyValueComparisonFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleCQKeyValueComparisonFilter.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Writables;
-
import org.apache.phoenix.expression.Expression;
@@ -32,8 +31,6 @@ import org.apache.phoenix.expression.Expression;
* part of the key value since the column qualifier is unique across all column
* families.
*
- *
- * @since 0.1
*/
public class SingleCQKeyValueComparisonFilter extends SingleKeyValueComparisonFilter {
public SingleCQKeyValueComparisonFilter() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/267715aa/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java
index 8929f8a..eaf8d35 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java
@@ -21,7 +21,6 @@ import java.io.DataInput;
import java.io.IOException;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.expression.KeyValueColumnExpression;
@@ -37,8 +36,6 @@ import org.apache.phoenix.schema.tuple.SingleKeyValueTuple;
* but for general expression evaluation in the case where only a single KeyValue
* column is referenced in the expression.
*
- *
- * @since 0.1
*/
public abstract class SingleKeyValueComparisonFilter extends BooleanExpressionFilter {
private final SingleKeyValueTuple inputTuple = new SingleKeyValueTuple();
@@ -76,8 +73,7 @@ public abstract class SingleKeyValueComparisonFilter extends BooleanExpressionFi
public ReturnCode filterKeyValue(Cell keyValue) {
if (this.matchedColumn) {
// We already found and matched the single column, all keys now pass
- // TODO: why won't this cause earlier versions of a kv to be included?
- return ReturnCode.INCLUDE;
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
}
if (this.foundColumn()) {
// We found all the columns, but did not match the expression, so skip to next row
@@ -87,19 +83,18 @@ public abstract class SingleKeyValueComparisonFilter extends BooleanExpressionFi
keyValue.getQualifierArray(), keyValue.getQualifierOffset(), keyValue.getQualifierLength()) != 0) {
// Remember the key in case this is the only key value we see.
// We'll need it if we have row key columns too.
- inputTuple.setKey(KeyValueUtil.ensureKeyValue(keyValue));
+ inputTuple.setKey(keyValue);
// This is a key value we're not interested in
- // TODO: use NEXT_COL when bug fix comes through that includes the row still
- return ReturnCode.INCLUDE;
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
}
- inputTuple.setKeyValue(KeyValueUtil.ensureKeyValue(keyValue));
+ inputTuple.setCell(keyValue);
// We have the columns, so evaluate here
if (!Boolean.TRUE.equals(evaluate(inputTuple))) {
return ReturnCode.NEXT_ROW;
}
this.matchedColumn = true;
- return ReturnCode.INCLUDE;
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
}
@Override
@@ -124,7 +119,7 @@ public abstract class SingleKeyValueComparisonFilter extends BooleanExpressionFi
return true;
}
- @Override
+ @Override
public void reset() {
inputTuple.reset();
matchedColumn = false;
@@ -137,7 +132,7 @@ public abstract class SingleKeyValueComparisonFilter extends BooleanExpressionFi
init();
}
- @SuppressWarnings("all") // suppressing missing @Override since this doesn't exist for HBase 0.94.4
+ @Override
public boolean isFamilyEssential(byte[] name) {
// Only the column families involved in the expression are essential.
// The others are for columns projected in the select expression
http://git-wip-us.apache.org/repos/asf/phoenix/blob/267715aa/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
index 667b3d7..77b4cf6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
@@ -274,7 +274,7 @@ public class SkipScanFilter extends FilterBase implements Writable {
// more than we need. We can optimize this by tracking whether each range in each slot position
// intersects.
ReturnCode endCode = navigate(upperExclusiveKey, 0, upperExclusiveKey.length, Terminate.AT);
- if (endCode == ReturnCode.INCLUDE) {
+ if (endCode == ReturnCode.INCLUDE || endCode == ReturnCode.INCLUDE_AND_NEXT_COL) {
setStartKey();
// If the upperExclusiveKey is equal to the start key, we've gone one position too far, since
// our upper key is exclusive. In that case, go to the previous key
@@ -358,7 +358,7 @@ public class SkipScanFilter extends FilterBase implements Writable {
// First check to see if we're in-range until we reach our end key
if (endKeyLength > 0) {
if (Bytes.compareTo(currentKey, offset, length, endKey, 0, endKeyLength) < 0) {
- return ReturnCode.INCLUDE;
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
}
// If key range of last slot is a single key, we can increment our position
@@ -490,7 +490,7 @@ public class SkipScanFilter extends FilterBase implements Writable {
// up to the upper range of our last slot. We do this for ranges and single keys
// since we potentially have multiple key values for the same row key.
setEndKey(ptr, minOffset, i);
- return ReturnCode.INCLUDE;
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
}
private boolean allTrailingNulls(int i) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/267715aa/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/SingleKeyValueTuple.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/SingleKeyValueTuple.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/SingleKeyValueTuple.java
index 8226208..e35eb13 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/SingleKeyValueTuple.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/SingleKeyValueTuple.java
@@ -17,24 +17,24 @@
*/
package org.apache.phoenix.schema.tuple;
-import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
public class SingleKeyValueTuple extends BaseTuple {
private static final byte[] UNITIALIZED_KEY_BUFFER = new byte[0];
- private KeyValue keyValue;
+ private Cell cell;
private ImmutableBytesWritable keyPtr = new ImmutableBytesWritable(UNITIALIZED_KEY_BUFFER);
public SingleKeyValueTuple() {
}
- public SingleKeyValueTuple(KeyValue keyValue) {
- if (keyValue == null) {
+ public SingleKeyValueTuple(Cell cell) {
+ if (cell == null) {
throw new NullPointerException();
}
- setKeyValue(keyValue);
+ setCell(cell);
}
public boolean hasKey() {
@@ -42,28 +42,27 @@ public class SingleKeyValueTuple extends BaseTuple {
}
public void reset() {
- this.keyValue = null;
+ this.cell = null;
keyPtr.set(UNITIALIZED_KEY_BUFFER);
}
- public void setKeyValue(KeyValue keyValue) {
- if (keyValue == null) {
+ public void setCell(Cell cell) {
+ if (cell == null) {
throw new IllegalArgumentException();
}
- this.keyValue = keyValue;
- setKey(keyValue);
+ this.cell = cell;
+ setKey(cell);
}
public void setKey(ImmutableBytesWritable ptr) {
keyPtr.set(ptr.get(), ptr.getOffset(), ptr.getLength());
}
- @SuppressWarnings("deprecation")
- public void setKey(KeyValue keyValue) {
- if (keyValue == null) {
+ public void setKey(Cell cell) {
+ if (cell == null) {
throw new IllegalArgumentException();
}
- keyPtr.set(keyValue.getBuffer(), keyValue.getRowOffset(), keyValue.getRowLength());
+ keyPtr.set(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
}
@Override
@@ -72,8 +71,8 @@ public class SingleKeyValueTuple extends BaseTuple {
}
@Override
- public KeyValue getValue(byte[] cf, byte[] cq) {
- return keyValue;
+ public Cell getValue(byte[] cf, byte[] cq) {
+ return cell;
}
@Override
@@ -83,29 +82,28 @@ public class SingleKeyValueTuple extends BaseTuple {
@Override
public String toString() {
- return "SingleKeyValueTuple[" + keyValue == null ? keyPtr.get() == UNITIALIZED_KEY_BUFFER ? "null" : Bytes.toStringBinary(keyPtr.get(),keyPtr.getOffset(),keyPtr.getLength()) : keyValue.toString() + "]";
+ return "SingleKeyValueTuple[" + cell == null ? keyPtr.get() == UNITIALIZED_KEY_BUFFER ? "null" : Bytes.toStringBinary(keyPtr.get(),keyPtr.getOffset(),keyPtr.getLength()) : cell.toString() + "]";
}
@Override
public int size() {
- return keyValue == null ? 0 : 1;
+ return cell == null ? 0 : 1;
}
@Override
- public KeyValue getValue(int index) {
- if (index != 0 || keyValue == null) {
+ public Cell getValue(int index) {
+ if (index != 0 || cell == null) {
throw new IndexOutOfBoundsException(Integer.toString(index));
}
- return keyValue;
+ return cell;
}
- @SuppressWarnings("deprecation")
@Override
public boolean getValue(byte[] family, byte[] qualifier,
ImmutableBytesWritable ptr) {
- if (keyValue == null)
+ if (cell == null)
return false;
- ptr.set(keyValue.getBuffer(), keyValue.getValueOffset(), keyValue.getValueLength());
+ ptr.set(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
return true;
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/267715aa/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java
index 898f778..4cb71ff 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java
@@ -402,7 +402,7 @@ public class SkipScanFilterTest extends TestCase {
skipper.reset();
assertFalse(skipper.filterAllRemaining());
assertFalse(skipper.filterRowKey(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength()));
- assertEquals(kv.toString(), ReturnCode.INCLUDE, skipper.filterKeyValue(kv));
+ assertEquals(kv.toString(), ReturnCode.INCLUDE_AND_NEXT_COL, skipper.filterKeyValue(kv));
}
@Override public String toString() {
[44/50] [abbrv] phoenix git commit: PHOENIX-2257 Failing integration
test in DerivedTableIT (addendum)
Posted by sa...@apache.org.
PHOENIX-2257 Failing integration test in DerivedTableIT (addendum)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/316871ba
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/316871ba
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/316871ba
Branch: refs/heads/4.x-HBase-1.0
Commit: 316871ba47a53696cdbfd4dbf6634e1eadaef013
Parents: a593e98
Author: maryannxue <ma...@gmail.com>
Authored: Tue Jan 5 09:36:36 2016 -0500
Committer: maryannxue <ma...@gmail.com>
Committed: Tue Jan 5 09:36:36 2016 -0500
----------------------------------------------------------------------
.../java/org/apache/phoenix/end2end/DerivedTableIT.java | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/316871ba/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java
index 95a7fed..82a742b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java
@@ -104,7 +104,8 @@ public class DerivedTableIT extends BaseClientManagedTimeIT {
"CLIENT MERGE SORT\n" +
"CLIENT SORTED BY [A]\n" +
"CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" +
- "CLIENT DISTINCT ON [COLLECTDISTINCT(B)]"}});
+ "CLIENT DISTINCT ON [COLLECTDISTINCT(B)]\n" +
+ "CLIENT SORTED BY [A DESC]"}});
testCases.add(new String[][] {
{}, {
"CLIENT PARALLEL 4-WAY FULL SCAN OVER ATABLE\n" +
@@ -120,7 +121,8 @@ public class DerivedTableIT extends BaseClientManagedTimeIT {
"CLIENT MERGE SORT\n" +
"CLIENT SORTED BY [A]\n" +
"CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" +
- "CLIENT DISTINCT ON [COLLECTDISTINCT(B)]"}});
+ "CLIENT DISTINCT ON [COLLECTDISTINCT(B)]\n" +
+ "CLIENT SORTED BY [A DESC]"}});
return testCases;
}
@@ -311,8 +313,8 @@ public class DerivedTableIT extends BaseClientManagedTimeIT {
rs = conn.createStatement().executeQuery("EXPLAIN " + query);
assertEquals(plans[0], QueryUtil.getExplainPlan(rs));
- // distinct b (groupby a, b) groupby a
- query = "SELECT DISTINCT COLLECTDISTINCT(t.b) FROM (SELECT b_string b, a_string a FROM aTable GROUP BY a_string, b_string) AS t GROUP BY t.a";
+ // distinct b (groupby a, b) groupby a orderby a
+ query = "SELECT DISTINCT COLLECTDISTINCT(t.b) FROM (SELECT b_string b, a_string a FROM aTable GROUP BY a_string, b_string) AS t GROUP BY t.a ORDER BY t.a DESC";
statement = conn.prepareStatement(query);
rs = statement.executeQuery();
assertTrue (rs.next());
[20/50] [abbrv] phoenix git commit: PHOENIX-1261 Update stats table
asynchronously
Posted by sa...@apache.org.
PHOENIX-1261 Update stats table asynchronously
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/08aa07f5
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/08aa07f5
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/08aa07f5
Branch: refs/heads/4.x-HBase-1.0
Commit: 08aa07f55395eb0d1195ef5646a4669947b4df05
Parents: 7c76bba
Author: Samarth <sa...@apache.org>
Authored: Tue Dec 22 13:56:56 2015 -0800
Committer: Samarth <sa...@apache.org>
Committed: Tue Dec 22 13:56:56 2015 -0800
----------------------------------------------------------------------
.../StatisticsCollectionRunTrackerIT.java | 167 ++++++++
.../phoenix/end2end/StatsCollectorIT.java | 6 +-
.../org/apache/phoenix/cache/GlobalCache.java | 7 +-
.../coprocessor/BaseScannerRegionObserver.java | 3 +-
.../UngroupedAggregateRegionObserver.java | 378 +++++++++++++------
.../org/apache/phoenix/query/QueryServices.java | 3 +
.../phoenix/query/QueryServicesOptions.java | 17 +
.../apache/phoenix/schema/MetaDataClient.java | 8 +-
.../stats/StatisticsCollectionRunTracker.java | 130 +++++++
.../schema/stats/StatisticsCollector.java | 8 +-
.../phoenix/schema/stats/StatisticsScanner.java | 121 +++---
.../phoenix/schema/stats/StatisticsWriter.java | 3 -
.../phoenix/query/QueryServicesTestImpl.java | 6 +-
13 files changed, 684 insertions(+), 173 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/08aa07f5/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java
new file mode 100644
index 0000000..c64038e
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.coprocessor;
+
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.COMPACTION_UPDATE_STATS_ROW_COUNT;
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.CONCURRENT_UPDATE_STATS_ROW_COUNT;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.end2end.BaseOwnClusterHBaseManagedTimeIT;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+public class StatisticsCollectionRunTrackerIT extends BaseOwnClusterHBaseManagedTimeIT {
+ @BeforeClass
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
+ // Must update config before starting server
+ props.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
+ props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(1024));
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @Test
+ public void testStateBeforeAndAfterUpdateStatsCommand() throws Exception {
+ String tableName = "testStateBeforeAndAfterUpdateStatsCommand".toUpperCase();
+ HRegionInfo regionInfo = createTableAndGetRegion(tableName);
+ StatisticsCollectionRunTracker tracker =
+ StatisticsCollectionRunTracker.getInstance(new Configuration());
+ // assert that the region wasn't added to the tracker
+ assertTrue(tracker.addUpdateStatsCommandRegion(regionInfo));
+ // assert that removing the region from the tracker works
+ assertTrue(tracker.removeUpdateStatsCommandRegion(regionInfo));
+ runUpdateStats(tableName);
+ // assert that after update stats is complete, tracker isn't tracking the region any more
+ assertFalse(tracker.removeUpdateStatsCommandRegion(regionInfo));
+ }
+
+ @Test
+ public void testStateBeforeAndAfterMajorCompaction() throws Exception {
+ String tableName = "testStateBeforeAndAfterMajorCompaction".toUpperCase();
+ HRegionInfo regionInfo = createTableAndGetRegion(tableName);
+ StatisticsCollectionRunTracker tracker =
+ StatisticsCollectionRunTracker.getInstance(new Configuration());
+ // Upsert values in the table.
+ String keyPrefix = "aaaaaaaaaaaaaaaaaaaa";
+ String upsert = "UPSERT INTO " + tableName + " VALUES (?, ?)";
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ PreparedStatement stmt = conn.prepareStatement(upsert);
+ for (int i = 0; i < 1000; i++) {
+ stmt.setString(1, keyPrefix + i);
+ stmt.setString(2, "KV" + i);
+ stmt.executeUpdate();
+ }
+ conn.commit();
+ }
+ // assert that the region wasn't added to the tracker
+ assertTrue(tracker.addCompactingRegion(regionInfo));
+ // assert that removing the region from the tracker works
+ assertTrue(tracker.removeCompactingRegion(regionInfo));
+ runMajorCompaction(tableName);
+
+ // assert that after major compaction is complete, tracker isn't tracking the region any more
+ assertFalse(tracker.removeCompactingRegion(regionInfo));
+ }
+
+ @Test
+ public void testMajorCompactionPreventsUpdateStatsFromRunning() throws Exception {
+ String tableName = "testMajorCompactionPreventsUpdateStatsFromRunning".toUpperCase();
+ HRegionInfo regionInfo = createTableAndGetRegion(tableName);
+ // simulate stats collection via major compaction by marking the region as compacting in the tracker
+ markRegionAsCompacting(regionInfo);
+ Assert.assertEquals("Row count didn't match", COMPACTION_UPDATE_STATS_ROW_COUNT, runUpdateStats(tableName));
+ StatisticsCollectionRunTracker tracker =
+ StatisticsCollectionRunTracker.getInstance(new Configuration());
+ // assert that the tracker state was cleared.
+ assertFalse(tracker.removeUpdateStatsCommandRegion(regionInfo));
+ }
+
+ @Test
+ public void testUpdateStatsPreventsAnotherUpdateStatsFromRunning() throws Exception {
+ String tableName = "testUpdateStatsPreventsAnotherUpdateStatsFromRunning".toUpperCase();
+ HRegionInfo regionInfo = createTableAndGetRegion(tableName);
+ markRunningUpdateStats(regionInfo);
+ Assert.assertEquals("Row count didn't match", CONCURRENT_UPDATE_STATS_ROW_COUNT,
+ runUpdateStats(tableName));
+ StatisticsCollectionRunTracker tracker =
+ StatisticsCollectionRunTracker.getInstance(new Configuration());
+ // assert that running the concurrent and race-losing update stats didn't clear the region
+ // from the tracker. If the method returned true it means the tracker was still tracking
+ // the region. Slightly counter-intuitive, yes.
+ assertTrue(tracker.removeUpdateStatsCommandRegion(regionInfo));
+ }
+
+ private void markRegionAsCompacting(HRegionInfo regionInfo) {
+ StatisticsCollectionRunTracker tracker =
+ StatisticsCollectionRunTracker.getInstance(new Configuration());
+ tracker.addCompactingRegion(regionInfo);
+ }
+
+ private void markRunningUpdateStats(HRegionInfo regionInfo) {
+ StatisticsCollectionRunTracker tracker =
+ StatisticsCollectionRunTracker.getInstance(new Configuration());
+ tracker.addUpdateStatsCommandRegion(regionInfo);
+ }
+
+ private HRegionInfo createTableAndGetRegion(String tableName) throws Exception {
+ byte[] tableNameBytes = Bytes.toBytes(tableName);
+ String ddl = "CREATE TABLE " + tableName + " (PK1 VARCHAR PRIMARY KEY, KV1 VARCHAR)";
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.createStatement().execute(ddl);
+ PhoenixConnection phxConn = conn.unwrap(PhoenixConnection.class);
+ try (HBaseAdmin admin = phxConn.getQueryServices().getAdmin()) {
+ List<HRegionInfo> tableRegions = admin.getTableRegions(tableNameBytes);
+ return tableRegions.get(0);
+ }
+ }
+ }
+
+ private long runUpdateStats(String tableName) throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ return conn.createStatement().executeUpdate("UPDATE STATISTICS " + tableName);
+ }
+ }
+
+ private void runMajorCompaction(String tableName) throws Exception {
+ try (PhoenixConnection conn = DriverManager.getConnection(getUrl()).unwrap(PhoenixConnection.class)) {
+ try (HBaseAdmin admin = conn.getQueryServices().getAdmin()) {
+ admin.flush(tableName);
+ admin.majorCompact(tableName);
+ Thread.sleep(10000);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/08aa07f5/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
index 23859d6..5e048c5 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
@@ -35,6 +35,7 @@ import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.ConnectionQueryServices;
@@ -241,8 +242,9 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
ConnectionQueryServices services = conn.unwrap(PhoenixConnection.class).getQueryServices();
HBaseAdmin admin = services.getAdmin();
try {
- admin.flush(tableName);
- admin.majorCompact(tableName);
+ TableName t = TableName.valueOf(tableName);
+ admin.flush(t);
+ admin.majorCompact(t);
Thread.sleep(10000); // FIXME: how do we know when compaction is done?
} finally {
admin.close();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/08aa07f5/phoenix-core/src/main/java/org/apache/phoenix/cache/GlobalCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/cache/GlobalCache.java b/phoenix-core/src/main/java/org/apache/phoenix/cache/GlobalCache.java
index 643112d..af5438c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/GlobalCache.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/GlobalCache.java
@@ -32,11 +32,9 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.memory.ChildMemoryManager;
import org.apache.phoenix.memory.GlobalMemoryManager;
-import org.apache.phoenix.parse.PFunction;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PMetaDataEntity;
-import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.util.SizedUtil;
import com.google.common.cache.Cache;
@@ -53,13 +51,13 @@ import com.google.common.cache.Weigher;
* @since 0.1
*/
public class GlobalCache extends TenantCacheImpl {
- private static GlobalCache INSTANCE;
+ private static volatile GlobalCache INSTANCE;
private final Configuration config;
// TODO: Use Guava cache with auto removal after lack of access
private final ConcurrentMap<ImmutableBytesWritable,TenantCache> perTenantCacheMap = new ConcurrentHashMap<ImmutableBytesWritable,TenantCache>();
// Cache for lastest PTable for a given Phoenix table
- private Cache<ImmutableBytesPtr,PMetaDataEntity> metaDataCache;
+ private volatile Cache<ImmutableBytesPtr,PMetaDataEntity> metaDataCache;
public void clearTenantCache() {
perTenantCacheMap.clear();
@@ -177,4 +175,5 @@ public class GlobalCache extends TenantCacheImpl {
return super.hashCode();
}
}
+
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/08aa07f5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index f98499f..5ab432c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -92,6 +92,7 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
public static final String GUIDEPOST_PER_REGION = "_GUIDEPOST_PER_REGION";
public static final String UPGRADE_DESC_ROW_KEY = "_UPGRADE_DESC_ROW_KEY";
public static final String SCAN_REGION_SERVER = "_SCAN_REGION_SERVER";
+ public static final String RUN_UPDATE_STATS_ASYNC = "_RunUpdateStatsAsync";
/**
* Attribute name used to pass custom annotations in Scans and Mutations (later). Custom annotations
@@ -169,7 +170,7 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
if (!isRegionObserverFor(scan)) {
return s;
}
- boolean success =false;
+ boolean success = false;
// Save the current span. When done with the child span, reset the span back to
// what it was. Otherwise, this causes the thread local storing the current span
// to not be reset back to null causing catastrophic infinite loops
http://git-wip-us.apache.org/repos/asf/phoenix/blob/08aa07f5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index a19cac9..562e904 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -21,7 +21,11 @@ import static org.apache.phoenix.query.QueryConstants.AGG_TIMESTAMP;
import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN;
import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY;
import static org.apache.phoenix.query.QueryConstants.UNGROUPED_AGG_ROW_KEY;
+import static org.apache.phoenix.query.QueryServices.COMMIT_STATS_ASYNC;
import static org.apache.phoenix.query.QueryServices.MUTATE_BATCH_SIZE_ATTRIB;
+import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_COMMIT_STATS_ASYNC;
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.COMPACTION_UPDATE_STATS_ROW_COUNT;
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.CONCURRENT_UPDATE_STATS_ROW_COUNT;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -35,7 +39,11 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.Callable;
+import co.cask.tephra.TxConstants;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.HConstants;
@@ -88,6 +96,7 @@ import org.apache.phoenix.schema.PTableImpl;
import org.apache.phoenix.schema.RowKeySchema;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.ValueSchema.Field;
+import org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker;
import org.apache.phoenix.schema.stats.StatisticsCollector;
import org.apache.phoenix.schema.tuple.MultiKeyValueTuple;
import org.apache.phoenix.schema.types.PBinary;
@@ -95,6 +104,7 @@ import org.apache.phoenix.schema.types.PChar;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PDouble;
import org.apache.phoenix.schema.types.PFloat;
+import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.KeyValueUtil;
@@ -108,19 +118,18 @@ import org.apache.phoenix.util.TimeKeeper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import co.cask.tephra.TxConstants;
-
/**
* Region observer that aggregates ungrouped rows(i.e. SQL query with aggregation function and no GROUP BY).
- *
- *
+ *
+ *
* @since 0.1
*/
-public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
+public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver {
// TODO: move all constants into a single class
public static final String UNGROUPED_AGG = "UngroupedAgg";
public static final String DELETE_AGG = "DeleteAgg";
@@ -131,7 +140,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
public static final String EMPTY_CF = "EmptyCF";
private static final Logger logger = LoggerFactory.getLogger(UngroupedAggregateRegionObserver.class);
private KeyValueBuilder kvBuilder;
-
+
@Override
public void start(CoprocessorEnvironment e) throws IOException {
super.start(e);
@@ -141,28 +150,28 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
}
private static void commitBatch(HRegion region, List<Mutation> mutations, byte[] indexUUID) throws IOException {
- if (indexUUID != null) {
- for (Mutation m : mutations) {
- m.setAttribute(PhoenixIndexCodec.INDEX_UUID, indexUUID);
- }
- }
- Mutation[] mutationArray = new Mutation[mutations.size()];
- // TODO: should we use the one that is all or none?
- logger.warn("Committing bactch of " + mutations.size() + " mutations for " + region.getRegionInfo().getTable().getNameAsString());
- region.batchMutate(mutations.toArray(mutationArray));
+ if (indexUUID != null) {
+ for (Mutation m : mutations) {
+ m.setAttribute(PhoenixIndexCodec.INDEX_UUID, indexUUID);
+ }
+ }
+ Mutation[] mutationArray = new Mutation[mutations.size()];
+ // TODO: should we use the one that is all or none?
+ logger.warn("Committing bactch of " + mutations.size() + " mutations for " + region.getRegionInfo().getTable().getNameAsString());
+ region.batchMutate(mutations.toArray(mutationArray), HConstants.NO_NONCE, HConstants.NO_NONCE);
}
public static void serializeIntoScan(Scan scan) {
scan.setAttribute(BaseScannerRegionObserver.UNGROUPED_AGG, QueryConstants.TRUE);
}
-
+
@Override
public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s)
throws IOException {
s = super.preScannerOpen(e, scan, s);
if (ScanUtil.isAnalyzeTable(scan)) {
// We are setting the start row and stop row such that it covers the entire region. As part
- // of Phonenix-1263 we are storing the guideposts against the physical table rather than
+ // of Phonenix-1263 we are storing the guideposts against the physical table rather than
// individual tenant specific tables.
scan.setStartRow(HConstants.EMPTY_START_ROW);
scan.setStopRow(HConstants.EMPTY_END_ROW);
@@ -170,17 +179,22 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
}
return s;
}
-
+
@Override
protected RegionScanner doPostScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c, final Scan scan, final RegionScanner s) throws IOException {
- HRegion region = c.getEnvironment().getRegion();
+ RegionCoprocessorEnvironment env = c.getEnvironment();
+ HRegion region = env.getRegion();
long ts = scan.getTimeRange().getMax();
- StatisticsCollector stats = null;
- if(ScanUtil.isAnalyzeTable(scan)) {
- byte[] gp_width_bytes = scan.getAttribute(BaseScannerRegionObserver.GUIDEPOST_WIDTH_BYTES);
- byte[] gp_per_region_bytes = scan.getAttribute(BaseScannerRegionObserver.GUIDEPOST_PER_REGION);
+ if (ScanUtil.isAnalyzeTable(scan)) {
+ byte[] gp_width_bytes =
+ scan.getAttribute(BaseScannerRegionObserver.GUIDEPOST_WIDTH_BYTES);
+ byte[] gp_per_region_bytes =
+ scan.getAttribute(BaseScannerRegionObserver.GUIDEPOST_PER_REGION);
// Let this throw, as this scan is being done for the sole purpose of collecting stats
- stats = new StatisticsCollector(c.getEnvironment(), region.getRegionInfo().getTable().getNameAsString(), ts, gp_width_bytes, gp_per_region_bytes);
+ StatisticsCollector statsCollector =
+ new StatisticsCollector(env, region.getRegionInfo().getTable()
+ .getNameAsString(), ts, gp_width_bytes, gp_per_region_bytes);
+ return collectStats(s, statsCollector, region, scan, env.getConfiguration());
}
int offsetToBe = 0;
if (ScanUtil.isLocalIndex(scan)) {
@@ -248,7 +262,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
if ((localIndexScan && !isDelete && !isDescRowKeyOrderUpgrade) || (j == null && p != null)) {
if (dataColumns != null) {
tupleProjector = IndexUtil.getTupleProjector(scan, dataColumns);
- dataRegion = IndexUtil.getDataRegion(c.getEnvironment());
+ dataRegion = IndexUtil.getDataRegion(env);
viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan);
}
ImmutableBytesWritable tempPtr = new ImmutableBytesWritable();
@@ -258,7 +272,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
}
if (j != null) {
- theScanner = new HashJoinRegionScanner(theScanner, p, j, ScanUtil.getTenantId(scan), c.getEnvironment());
+ theScanner = new HashJoinRegionScanner(theScanner, p, j, ScanUtil.getTenantId(scan), env);
}
int batchSize = 0;
@@ -267,22 +281,20 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
if (isDescRowKeyOrderUpgrade || isDelete || isUpsert || (deleteCQ != null && deleteCF != null) || emptyCF != null || buildLocalIndex) {
// TODO: size better
mutations = Lists.newArrayListWithExpectedSize(1024);
- batchSize = c.getEnvironment().getConfiguration().getInt(MUTATE_BATCH_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE);
+ batchSize = env.getConfiguration().getInt(MUTATE_BATCH_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE);
}
Aggregators aggregators = ServerAggregators.deserialize(
- scan.getAttribute(BaseScannerRegionObserver.AGGREGATORS), c.getEnvironment().getConfiguration());
+ scan.getAttribute(BaseScannerRegionObserver.AGGREGATORS), env.getConfiguration());
Aggregator[] rowAggregators = aggregators.getAggregators();
boolean hasMore;
boolean hasAny = false;
MultiKeyValueTuple result = new MultiKeyValueTuple();
if (logger.isDebugEnabled()) {
- logger.debug(LogUtil.addCustomAnnotations("Starting ungrouped coprocessor scan " + scan + " "+region.getRegionInfo(), ScanUtil.getCustomAnnotations(scan)));
+ logger.debug(LogUtil.addCustomAnnotations("Starting ungrouped coprocessor scan " + scan + " "+region.getRegionInfo(), ScanUtil.getCustomAnnotations(scan)));
}
long rowCount = 0;
final RegionScanner innerScanner = theScanner;
region.startRegionOperation();
- boolean updateStats = stats != null;
- boolean success = false;
try {
synchronized (innerScanner) {
do {
@@ -291,9 +303,6 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
// since this is an indication of whether or not there are more values after the
// ones returned
hasMore = innerScanner.nextRaw(results);
- if (updateStats) {
- stats.collectStatistics(results);
- }
if (!results.isEmpty()) {
rowCount++;
result.setKeyValues(results);
@@ -313,8 +322,8 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
// Special case for re-writing DESC ARRAY, as the actual byte value needs to change in this case
if (field.getDataType().isArrayType()) {
field.getDataType().coerceBytes(ptr, null, field.getDataType(),
- field.getMaxLength(), field.getScale(), field.getSortOrder(),
- field.getMaxLength(), field.getScale(), field.getSortOrder(), true); // force to use correct separator byte
+ field.getMaxLength(), field.getScale(), field.getSortOrder(),
+ field.getMaxLength(), field.getScale(), field.getSortOrder(), true); // force to use correct separator byte
}
// Special case for re-writing DESC CHAR or DESC BINARY, to force the re-writing of trailing space characters
else if (field.getDataType() == PChar.INSTANCE || field.getDataType() == PBinary.INSTANCE) {
@@ -323,7 +332,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
len--;
}
ptr.set(ptr.get(), ptr.getOffset(), len);
- // Special case for re-writing DESC FLOAT and DOUBLE, as they're not inverted like they should be (PHOENIX-2171)
+ // Special case for re-writing DESC FLOAT and DOUBLE, as they're not inverted like they should be (PHOENIX-2171)
} else if (field.getDataType() == PFloat.INSTANCE || field.getDataType() == PDouble.INSTANCE) {
byte[] invertedBytes = SortOrder.invert(ptr.get(), ptr.getOffset(), ptr.getLength());
ptr.set(invertedBytes);
@@ -342,8 +351,8 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
}
writeToTable.newKey(ptr, values);
if (Bytes.compareTo(
- firstKV.getRowArray(), firstKV.getRowOffset() + offset, firstKV.getRowLength(),
- ptr.get(),ptr.getOffset() + offset,ptr.getLength()) == 0) {
+ firstKV.getRowArray(), firstKV.getRowOffset() + offset, firstKV.getRowLength(),
+ ptr.get(),ptr.getOffset() + offset,ptr.getLength()) == 0) {
continue;
}
byte[] newRow = ByteUtil.copyKeyBytesIfNecessary(ptr);
@@ -357,21 +366,21 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
for (Cell cell : results) {
// Copy existing cell but with new row key
Cell newCell = new KeyValue(newRow, 0, newRow.length,
- cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
- cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
- cell.getTimestamp(), KeyValue.Type.codeToType(cell.getTypeByte()),
- cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
+ cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
+ cell.getTimestamp(), KeyValue.Type.codeToType(cell.getTypeByte()),
+ cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
switch (KeyValue.Type.codeToType(cell.getTypeByte())) {
case Put:
// If Put, point delete old Put
Delete del = new Delete(oldRow);
del.addDeleteMarker(new KeyValue(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
- cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
- cell.getQualifierArray(), cell.getQualifierOffset(),
- cell.getQualifierLength(), cell.getTimestamp(), KeyValue.Type.Delete,
- ByteUtil.EMPTY_BYTE_ARRAY, 0, 0));
+ cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
+ cell.getQualifierArray(), cell.getQualifierOffset(),
+ cell.getQualifierLength(), cell.getTimestamp(), KeyValue.Type.Delete,
+ ByteUtil.EMPTY_BYTE_ARRAY, 0, 0));
mutations.add(del);
-
+
Put put = new Put(newRow);
put.add(newCell);
mutations.add(put);
@@ -391,13 +400,13 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
if (!results.isEmpty()) {
result.getKey(ptr);
ValueGetter valueGetter =
- maintainer.createGetterFromKeyValues(
- ImmutableBytesPtr.copyBytesIfNecessary(ptr),
- results);
+ maintainer.createGetterFromKeyValues(
+ ImmutableBytesPtr.copyBytesIfNecessary(ptr),
+ results);
Put put = maintainer.buildUpdateMutation(kvBuilder,
valueGetter, ptr, ts,
- c.getEnvironment().getRegion().getStartKey(),
- c.getEnvironment().getRegion().getEndKey());
+ env.getRegion().getRegionInfo().getStartKey(),
+ env.getRegion().getRegionInfo().getEndKey());
indexMutations.add(put);
}
}
@@ -406,12 +415,12 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
// FIXME: the version of the Delete constructor without the lock
// args was introduced in 0.94.4, thus if we try to use it here
// we can no longer use the 0.94.2 version of the client.
- Cell firstKV = results.get(0);
- Delete delete = new Delete(firstKV.getRowArray(),
- firstKV.getRowOffset(), firstKV.getRowLength(),ts);
- mutations.add(delete);
- // force tephra to ignore this deletes
- delete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ Cell firstKV = results.get(0);
+ Delete delete = new Delete(firstKV.getRowArray(),
+ firstKV.getRowOffset(), firstKV.getRowLength(),ts);
+ mutations.add(delete);
+ // force tephra to ignore this deletes
+ delete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
} else if (isUpsert) {
Arrays.fill(values, null);
int i = 0;
@@ -423,7 +432,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
// If SortOrder from expression in SELECT doesn't match the
// column being projected into then invert the bits.
if (expression.getSortOrder() !=
- projectedColumns.get(i).getSortOrder()) {
+ projectedColumns.get(i).getSortOrder()) {
SortOrder.invert(values[i], 0, values[i], 0,
values[i].length);
}
@@ -436,13 +445,13 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
if (expression.evaluate(result, ptr)) {
PColumn column = projectedColumns.get(i);
Object value = expression.getDataType()
- .toObject(ptr, column.getSortOrder());
+ .toObject(ptr, column.getSortOrder());
// We are guaranteed that the two column will have the
// same type.
if (!column.getDataType().isSizeCompatible(ptr, value,
- column.getDataType(), expression.getMaxLength(),
- expression.getScale(), column.getMaxLength(),
- column.getScale())) {
+ column.getDataType(), expression.getMaxLength(),
+ expression.getScale(), column.getMaxLength(),
+ column.getScale())) {
throw new DataExceedsCapacityException(
column.getDataType(), column.getMaxLength(),
column.getScale());
@@ -486,7 +495,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
* We insert one empty key value per row per timestamp.
*/
Set<Long> timeStamps =
- Sets.newHashSetWithExpectedSize(results.size());
+ Sets.newHashSetWithExpectedSize(results.size());
for (Cell kv : results) {
long kvts = kv.getTimestamp();
if (!timeStamps.contains(kvts)) {
@@ -512,37 +521,25 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
} catch (ConstraintViolationException e) {
// Log and ignore in count
logger.error(LogUtil.addCustomAnnotations("Failed to create row in " +
- region.getRegionNameAsString() + " with values " +
- SchemaUtil.toString(values),
- ScanUtil.getCustomAnnotations(scan)), e);
+ region.getRegionInfo().getRegionNameAsString() + " with values " +
+ SchemaUtil.toString(values),
+ ScanUtil.getCustomAnnotations(scan)), e);
continue;
}
aggregators.aggregate(rowAggregators, result);
hasAny = true;
}
} while (hasMore);
- success = true;
}
} finally {
try {
- if (success && updateStats) {
- try {
- stats.updateStatistic(region);
- } finally {
- stats.close();
- }
- }
+ innerScanner.close();
} finally {
- try {
- innerScanner.close();
- } finally {
- region.closeRegionOperation();
- }
+ region.closeRegionOperation();
}
}
-
if (logger.isDebugEnabled()) {
- logger.debug(LogUtil.addCustomAnnotations("Finished scanning " + rowCount + " rows for ungrouped coprocessor scan " + scan, ScanUtil.getCustomAnnotations(scan)));
+ logger.debug(LogUtil.addCustomAnnotations("Finished scanning " + rowCount + " rows for ungrouped coprocessor scan " + scan, ScanUtil.getCustomAnnotations(scan)));
}
if (!mutations.isEmpty()) {
@@ -560,7 +557,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
keyValue = KeyValueUtil.newKeyValue(UNGROUPED_AGG_ROW_KEY, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, AGG_TIMESTAMP, value, 0, value.length);
}
final KeyValue aggKeyValue = keyValue;
-
+
RegionScanner scanner = new BaseRegionScanner() {
private boolean done = !hadAny;
@@ -586,10 +583,10 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
results.add(aggKeyValue);
return false;
}
-
+
@Override
public long getMaxResultSize() {
- return scan.getMaxResultSize();
+ return scan.getMaxResultSize();
}
};
return scanner;
@@ -610,7 +607,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
table = c.getEnvironment().getTable(indexTable);
table.batch(indexMutations);
} catch (InterruptedException ie) {
- ServerUtil.throwIOException(c.getEnvironment().getRegion().getRegionNameAsString(),
+ ServerUtil.throwIOException(c.getEnvironment().getRegion().getRegionInfo().getRegionNameAsString(),
ie);
} finally {
if (table != null) table.close();
@@ -618,7 +615,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
}
indexMutations.clear();
}
-
+
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c,
final Store store, InternalScanner scanner, final ScanType scanType)
@@ -643,7 +640,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
StatisticsCollector stats = new StatisticsCollector(
c.getEnvironment(), table.getNameAsString(),
clientTimeStamp, store.getFamily().getName());
- internalScanner = stats.createCompactionScanner(c.getEnvironment().getRegion(), store, scanner, mergeRegions);
+ internalScanner = stats.createCompactionScanner(c.getEnvironment(), store, scanner, mergeRegions);
} catch (IOException e) {
// If we can't reach the stats table, don't interrupt the normal
// compaction operation, just log a warning.
@@ -654,48 +651,209 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
}
return internalScanner;
}
-
-
+
+
@Override
public void postSplit(final ObserverContext<RegionCoprocessorEnvironment> e, final HRegion l,
final HRegion r) throws IOException {
- final HRegion region = e.getEnvironment().getRegion();
+ final Configuration config = e.getEnvironment().getConfiguration();
+ boolean async = config.getBoolean(COMMIT_STATS_ASYNC, DEFAULT_COMMIT_STATS_ASYNC);
+ if (!async) {
+ splitStatsInternal(e, l, r);
+ } else {
+ StatisticsCollectionRunTracker.getInstance(config).runTask(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ splitStatsInternal(e, l, r);
+ return null;
+ }
+ });
+ }
+ }
+
+ private void splitStatsInternal(final ObserverContext<RegionCoprocessorEnvironment> e,
+ final HRegion l, final HRegion r) {
+ final RegionCoprocessorEnvironment env = e.getEnvironment();
+ final HRegion region = env.getRegion();
final TableName table = region.getRegionInfo().getTable();
try {
- boolean useCurrentTime =
- e.getEnvironment().getConfiguration().getBoolean(QueryServices.STATS_USE_CURRENT_TIME_ATTRIB,
+ boolean useCurrentTime =
+ env.getConfiguration().getBoolean(QueryServices.STATS_USE_CURRENT_TIME_ATTRIB,
QueryServicesOptions.DEFAULT_STATS_USE_CURRENT_TIME);
// Provides a means of clients controlling their timestamps to not use current time
// when background tasks are updating stats. Instead we track the max timestamp of
// the cells and use that.
- final long clientTimeStamp = useCurrentTime ? TimeKeeper.SYSTEM.getCurrentTime() : StatisticsCollector.NO_TIMESTAMP;
+ final long clientTimeStamp = useCurrentTime ? TimeKeeper.SYSTEM.getCurrentTime() :
+ StatisticsCollector.NO_TIMESTAMP;
User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- StatisticsCollector stats = new StatisticsCollector(e.getEnvironment(),
- table.getNameAsString(), clientTimeStamp);
- try {
- stats.splitStats(region, l, r);
- return null;
- } finally {
- if (stats != null) stats.close();
- }
- }
- });
- } catch (IOException ioe) {
+ @Override
+ public Void run() throws Exception {
+ StatisticsCollector stats = new StatisticsCollector(env,
+ table.getNameAsString(), clientTimeStamp);
+ try {
+ stats.splitStats(region, l, r);
+ return null;
+ } finally {
+ if (stats != null) stats.close();
+ }
+ }
+ });
+ } catch (IOException ioe) {
if(logger.isWarnEnabled()) {
logger.warn("Error while collecting stats during split for " + table,ioe);
}
- }
+ }
}
+
private static PTable deserializeTable(byte[] b) {
try {
PTableProtos.PTable ptableProto = PTableProtos.PTable.parseFrom(b);
return PTableImpl.createFromProto(ptableProto);
} catch (IOException e) {
throw new RuntimeException(e);
- }
+ }
+ }
+
+ private RegionScanner collectStats(final RegionScanner innerScanner, StatisticsCollector stats,
+ final HRegion region, final Scan scan, Configuration config) throws IOException {
+ StatsCollectionCallable callable =
+ new StatsCollectionCallable(stats, region, innerScanner, config);
+ byte[] asyncBytes = scan.getAttribute(BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC);
+ boolean async = false;
+ if (asyncBytes != null) {
+ async = Bytes.toBoolean(asyncBytes);
+ }
+ long rowCount = 0; // in case of async, we report 0 as number of rows updated
+ StatisticsCollectionRunTracker statsRunTracker =
+ StatisticsCollectionRunTracker.getInstance(config);
+ boolean runUpdateStats = statsRunTracker.addUpdateStatsCommandRegion(region.getRegionInfo());
+ if (runUpdateStats) {
+ if (!async) {
+ rowCount = callable.call();
+ } else {
+ statsRunTracker.runTask(callable);
+ }
+ } else {
+ rowCount = CONCURRENT_UPDATE_STATS_ROW_COUNT;
+ logger.info("UPDATE STATISTICS didn't run because another UPDATE STATISTICS command was already running on the region "
+ + region.getRegionInfo().getRegionNameAsString());
+ }
+ byte[] rowCountBytes = PLong.INSTANCE.toBytes(Long.valueOf(rowCount));
+ final KeyValue aggKeyValue =
+ KeyValueUtil.newKeyValue(UNGROUPED_AGG_ROW_KEY, SINGLE_COLUMN_FAMILY,
+ SINGLE_COLUMN, AGG_TIMESTAMP, rowCountBytes, 0, rowCountBytes.length);
+ RegionScanner scanner = new BaseRegionScanner() {
+ @Override
+ public HRegionInfo getRegionInfo() {
+ return region.getRegionInfo();
+ }
+
+ @Override
+ public boolean isFilterDone() {
+ return true;
+ }
+
+ @Override
+ public void close() throws IOException {
+ // no-op because we want to manage closing of the inner scanner ourselves.
+ }
+
+ @Override
+ public boolean next(List<Cell> results) throws IOException {
+ results.add(aggKeyValue);
+ return false;
+ }
+
+ @Override
+ public long getMaxResultSize() {
+ return scan.getMaxResultSize();
+ }
+ };
+ return scanner;
+ }
+
+ /**
+ *
+ * Callable to encapsulate the collection of stats triggered by
+ * UPDATE STATISTICS command.
+ *
+ * Package private for tests.
+ */
+ static class StatsCollectionCallable implements Callable<Long> {
+ private final StatisticsCollector stats;
+ private final HRegion region;
+ private final RegionScanner innerScanner;
+ private final Configuration config;
+
+ StatsCollectionCallable(StatisticsCollector s, HRegion r, RegionScanner rs,
+ Configuration config) {
+ this.stats = s;
+ this.region = r;
+ this.innerScanner = rs;
+ this.config = config;
+ }
+
+ @Override
+ public Long call() throws IOException {
+ return collectStatsInternal();
+ }
+
+ private boolean areStatsBeingCollectedViaCompaction() {
+ return StatisticsCollectionRunTracker.getInstance(config)
+ .areStatsBeingCollectedOnCompaction(region.getRegionInfo());
+ }
+
+ private long collectStatsInternal() throws IOException {
+ long startTime = System.currentTimeMillis();
+ region.startRegionOperation();
+ boolean hasMore = false;
+ boolean noErrors = false;
+ boolean compactionRunning = areStatsBeingCollectedViaCompaction();
+ long rowCount = 0;
+ try {
+ if (!compactionRunning) {
+ synchronized (innerScanner) {
+ do {
+ List<Cell> results = new ArrayList<Cell>();
+ hasMore = innerScanner.nextRaw(results);
+ stats.collectStatistics(results);
+ rowCount++;
+ compactionRunning = areStatsBeingCollectedViaCompaction();
+ } while (hasMore && !compactionRunning);
+ noErrors = true;
+ }
+ }
+ return compactionRunning ? COMPACTION_UPDATE_STATS_ROW_COUNT : rowCount;
+ } catch (IOException e) {
+ logger.error("IOException in update stats: " + Throwables.getStackTraceAsString(e));
+ throw e;
+ } finally {
+ try {
+ if (noErrors && !compactionRunning) {
+ stats.updateStatistic(region);
+ logger.info("UPDATE STATISTICS finished successfully for scanner: "
+ + innerScanner + ". Number of rows scanned: " + rowCount
+ + ". Time: " + (System.currentTimeMillis() - startTime));
+ }
+ if (compactionRunning) {
+ logger.info("UPDATE STATISTICS stopped in between because major compaction was running for region "
+ + region.getRegionInfo().getRegionNameAsString());
+ }
+ } finally {
+ try {
+ StatisticsCollectionRunTracker.getInstance(config).removeUpdateStatsCommandRegion(region.getRegionInfo());
+ stats.close();
+ } finally {
+ try {
+ innerScanner.close();
+ } finally {
+ region.closeRegionOperation();
+ }
+ }
+ }
+ }
+ }
}
private static List<Expression> deserializeExpressions(byte[] b) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/08aa07f5/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 155ad3e..908c479 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -151,6 +151,9 @@ public interface QueryServices extends SQLCloseable {
public static final String STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB = "phoenix.stats.guidepost.width";
public static final String STATS_GUIDEPOST_PER_REGION_ATTRIB = "phoenix.stats.guidepost.per.region";
public static final String STATS_USE_CURRENT_TIME_ATTRIB = "phoenix.stats.useCurrentTime";
+ public static final String RUN_UPDATE_STATS_ASYNC = "phoenix.update.stats.command.async";
+ public static final String STATS_SERVER_POOL_SIZE = "phoenix.stats.pool.size";
+ public static final String COMMIT_STATS_ASYNC = "phoenix.stats.commit.async";
public static final String SEQUENCE_SALT_BUCKETS_ATTRIB = "phoenix.sequence.saltBuckets";
public static final String COPROCESSOR_PRIORITY_ATTRIB = "phoenix.coprocessor.priority";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/08aa07f5/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 0a0bf07..730a5ef 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -22,6 +22,7 @@ import static org.apache.phoenix.query.QueryServices.ALLOW_VIEWS_ADD_NEW_CF_BASE
import static org.apache.phoenix.query.QueryServices.CALL_QUEUE_PRODUCER_ATTRIB_NAME;
import static org.apache.phoenix.query.QueryServices.CALL_QUEUE_ROUND_ROBIN_ATTRIB;
import static org.apache.phoenix.query.QueryServices.COLLECT_REQUEST_LEVEL_METRICS;
+import static org.apache.phoenix.query.QueryServices.COMMIT_STATS_ASYNC;
import static org.apache.phoenix.query.QueryServices.DATE_FORMAT_ATTRIB;
import static org.apache.phoenix.query.QueryServices.DATE_FORMAT_TIMEZONE_ATTRIB;
import static org.apache.phoenix.query.QueryServices.DELAY_FOR_SCHEMA_UPDATE_CHECK;
@@ -54,6 +55,7 @@ import static org.apache.phoenix.query.QueryServices.REGIONSERVER_INFO_PORT_ATTR
import static org.apache.phoenix.query.QueryServices.REGIONSERVER_LEASE_PERIOD_ATTRIB;
import static org.apache.phoenix.query.QueryServices.ROW_KEY_ORDER_SALTED_TABLE_ATTRIB;
import static org.apache.phoenix.query.QueryServices.RPC_TIMEOUT_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.RUN_UPDATE_STATS_ASYNC;
import static org.apache.phoenix.query.QueryServices.SCAN_CACHE_SIZE_ATTRIB;
import static org.apache.phoenix.query.QueryServices.SCAN_RESULT_CHUNK_SIZE;
import static org.apache.phoenix.query.QueryServices.SEQUENCE_CACHE_SIZE_ATTRIB;
@@ -173,6 +175,9 @@ public class QueryServicesOptions {
// compression we're getting)
public static final long DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES = 3* 100 * 1024 *1024;
public static final boolean DEFAULT_STATS_USE_CURRENT_TIME = true;
+ public static final boolean DEFAULT_RUN_UPDATE_STATS_ASYNC = true;
+ public static final boolean DEFAULT_COMMIT_STATS_ASYNC = true;
+ public static final int DEFAULT_STATS_POOL_SIZE = 4;
public static final boolean DEFAULT_USE_REVERSE_SCAN = true;
@@ -248,6 +253,8 @@ public class QueryServicesOptions {
Configuration config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration();
QueryServicesOptions options = new QueryServicesOptions(config)
.setIfUnset(STATS_USE_CURRENT_TIME_ATTRIB, DEFAULT_STATS_USE_CURRENT_TIME)
+ .setIfUnset(RUN_UPDATE_STATS_ASYNC, DEFAULT_RUN_UPDATE_STATS_ASYNC)
+ .setIfUnset(COMMIT_STATS_ASYNC, DEFAULT_COMMIT_STATS_ASYNC)
.setIfUnset(KEEP_ALIVE_MS_ATTRIB, DEFAULT_KEEP_ALIVE_MS)
.setIfUnset(THREAD_POOL_SIZE_ATTRIB, DEFAULT_THREAD_POOL_SIZE)
.setIfUnset(QUEUE_SIZE_ATTRIB, DEFAULT_QUEUE_SIZE)
@@ -576,4 +583,14 @@ public class QueryServicesOptions {
config.set(EXTRA_JDBC_ARGUMENTS_ATTRIB, extraArgs);
return this;
}
+
+ public QueryServicesOptions setRunUpdateStatsAsync(boolean flag) {
+ config.setBoolean(RUN_UPDATE_STATS_ASYNC, flag);
+ return this;
+ }
+
+ public QueryServicesOptions setCommitStatsAsync(boolean flag) {
+ config.setBoolean(COMMIT_STATS_ASYNC, flag);
+ return this;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/08aa07f5/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 9c365ab..35d61ff 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -904,7 +904,13 @@ public class MetaDataClient {
}
}
}
- return new MutationState((int)rowCount, connection);
+ final long count = rowCount;
+ return new MutationState(1, connection) {
+ @Override
+ public long getUpdateCount() {
+ return count;
+ }
+ };
}
private long updateStatisticsInternal(PName physicalName, PTable logicalTable, Map<String, Object> statsProps) throws SQLException {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/08aa07f5/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java
new file mode 100644
index 0000000..5f6be3f
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema.stats;
+
+import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
+
+/**
+ * Singleton that is used to track state associated with regions undergoing stats collection at the
+ * region server's JVM level.
+ */
+public class StatisticsCollectionRunTracker {
+ private static volatile StatisticsCollectionRunTracker INSTANCE;
+ private final Set<HRegionInfo> updateStatsRegions = Collections
+ .newSetFromMap(new ConcurrentHashMap<HRegionInfo, Boolean>());
+ private final Set<HRegionInfo> compactingRegions = Collections
+ .newSetFromMap(new ConcurrentHashMap<HRegionInfo, Boolean>());
+ private final ExecutorService executor;
+
+ // Constants added for testing purposes
+ public static final long CONCURRENT_UPDATE_STATS_ROW_COUNT = -100l;
+ public static final long COMPACTION_UPDATE_STATS_ROW_COUNT = -200l;
+
+ public static StatisticsCollectionRunTracker getInstance(Configuration config) {
+ StatisticsCollectionRunTracker result = INSTANCE;
+ if (result == null) {
+ synchronized (StatisticsCollectionRunTracker.class) {
+ result = INSTANCE;
+ if (result == null) {
+ INSTANCE = result = new StatisticsCollectionRunTracker(config);
+ }
+ }
+ }
+ return result;
+ }
+
+ private StatisticsCollectionRunTracker(Configuration config) {
+ int poolSize =
+ config.getInt(QueryServices.STATS_SERVER_POOL_SIZE,
+ QueryServicesOptions.DEFAULT_STATS_POOL_SIZE);
+ executor = Executors.newFixedThreadPool(poolSize, new ThreadFactory() {
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = Executors.defaultThreadFactory().newThread(r);
+ t.setDaemon(true);
+ return t;
+ }
+ });
+ }
+
+ /**
+ * @param regionInfo for the region that should be marked as undergoing stats collection via
+ * major compaction.
+ * @return true if the region wasn't already marked for stats collection via compaction, false
+ * otherwise.
+ */
+ public boolean addCompactingRegion(HRegionInfo regionInfo) {
+ return compactingRegions.add(regionInfo);
+ }
+
+ /**
+ * @param regionInfo for the region that should be unmarked as undergoing stats collection via
+ * major compaction.
+ * @return true if the region was marked for stats collection via compaction, false otherwise.
+ */
+ public boolean removeCompactingRegion(HRegionInfo regionInfo) {
+ return compactingRegions.remove(regionInfo);
+ }
+
+ /**
+ * @param regionInfo for the region to check for.
+ * @return true if stats are being collected for the region via major compaction, false
+ * otherwise.
+ */
+ public boolean areStatsBeingCollectedOnCompaction(HRegionInfo regionInfo) {
+ return compactingRegions.contains(regionInfo);
+ }
+
+ /**
+ * @param regionInfo for the region to run UPDATE STATISTICS command on.
+ * @return true if UPDATE STATISTICS wasn't already running on the region, false otherwise.
+ */
+ public boolean addUpdateStatsCommandRegion(HRegionInfo regionInfo) {
+ return updateStatsRegions.add(regionInfo);
+ }
+
+ /**
+ * @param regionInfo for the region to mark as not running UPDATE STATISTICS command on.
+ * @return true if UPDATE STATISTICS was running on the region, false otherwise.
+ */
+ public boolean removeUpdateStatsCommandRegion(HRegionInfo regionInfo) {
+ return updateStatsRegions.remove(regionInfo);
+ }
+
+ /**
+ * Enqueues the task for execution.
+ * @param <T>
+ * @param c task to execute
+ */
+ public <T> Future<T> runTask(Callable<T> c) {
+ return executor.submit(c);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/08aa07f5/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
index 22fdf90..07fef1c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
@@ -216,14 +216,14 @@ public class StatisticsCollector {
}
}
- public InternalScanner createCompactionScanner(HRegion region, Store store, InternalScanner s,
+ public InternalScanner createCompactionScanner(RegionCoprocessorEnvironment env, Store store, InternalScanner s,
Pair<HRegionInfo, HRegionInfo> mergeRegions) throws IOException {
// See if this is for Major compaction
if (logger.isDebugEnabled()) {
logger.debug("Compaction scanner created for stats");
}
ImmutableBytesPtr cfKey = new ImmutableBytesPtr(store.getFamily().getName());
- return getInternalScanner(region, store, s, cfKey, mergeRegions);
+ return getInternalScanner(env, store, s, cfKey, mergeRegions);
}
public void splitStats(HRegion parent, HRegion left, HRegion right) {
@@ -245,9 +245,9 @@ public class StatisticsCollector {
}
}
- protected InternalScanner getInternalScanner(HRegion region, Store store, InternalScanner internalScan,
+ protected InternalScanner getInternalScanner(RegionCoprocessorEnvironment env, Store store, InternalScanner internalScan,
ImmutableBytesPtr family, Pair<HRegionInfo, HRegionInfo> mergeRegions) {
- return new StatisticsScanner(this, statsTable, region, internalScan, family, mergeRegions);
+ return new StatisticsScanner(this, statsTable, env, internalScan, family, mergeRegions);
}
public void clear() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/08aa07f5/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
index 5c460e3..550a772 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
@@ -17,21 +17,28 @@
*/
package org.apache.phoenix.schema.stats;
+import static org.apache.phoenix.query.QueryServices.COMMIT_STATS_ASYNC;
+import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_COMMIT_STATS_ASYNC;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.Callable;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+
/**
* The scanner that does the scanning to collect the stats during major compaction.{@link StatisticsCollector}
*/
@@ -43,15 +50,18 @@ public class StatisticsScanner implements InternalScanner {
private StatisticsCollector tracker;
private ImmutableBytesPtr family;
private Pair<HRegionInfo, HRegionInfo> mergeRegions;
+ private final Configuration config;
- public StatisticsScanner(StatisticsCollector tracker, StatisticsWriter stats, HRegion region,
+ public StatisticsScanner(StatisticsCollector tracker, StatisticsWriter stats, RegionCoprocessorEnvironment env,
InternalScanner delegate, ImmutableBytesPtr family, Pair<HRegionInfo, HRegionInfo> mergeRegions) {
this.tracker = tracker;
this.stats = stats;
this.delegate = delegate;
- this.region = region;
+ this.region = env.getRegion();
+ this.config = env.getConfiguration();
this.family = family;
this.mergeRegions = mergeRegions;
+ StatisticsCollectionRunTracker.getInstance(config).addCompactingRegion(region.getRegionInfo());
}
@Override
@@ -79,65 +89,82 @@ public class StatisticsScanner implements InternalScanner {
tracker.collectStatistics(results);
}
}
-
- @Override
- public void close() throws IOException {
- IOException toThrow = null;
- try {
- // update the statistics table
- // Just verify if this if fine
- ArrayList<Mutation> mutations = new ArrayList<Mutation>();
- if (mergeRegions != null) {
- if (mergeRegions.getFirst() != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Deleting stale stats for the region "
- + mergeRegions.getFirst().getRegionNameAsString() + " as part of major compaction");
+
+ private class StatisticsScannerCallable implements Callable<Void> {
+ @Override
+ public Void call() throws IOException {
+ IOException toThrow = null;
+ StatisticsCollectionRunTracker statsRunState =
+ StatisticsCollectionRunTracker.getInstance(config);
+ try {
+ // update the statistics table
+ // Just verify if this if fine
+ ArrayList<Mutation> mutations = new ArrayList<Mutation>();
+ if (mergeRegions != null) {
+ if (mergeRegions.getFirst() != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Deleting stale stats for the region "
+ + mergeRegions.getFirst().getRegionNameAsString() + " as part of major compaction");
+ }
+ stats.deleteStats(mergeRegions.getFirst().getRegionName(), tracker, family, mutations);
}
- stats.deleteStats(mergeRegions.getFirst().getRegionName(), tracker, family, mutations);
- }
- if (mergeRegions.getSecond() != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Deleting stale stats for the region "
- + mergeRegions.getSecond().getRegionNameAsString() + " as part of major compaction");
+ if (mergeRegions.getSecond() != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Deleting stale stats for the region "
+ + mergeRegions.getSecond().getRegionNameAsString() + " as part of major compaction");
+ }
+ stats.deleteStats(mergeRegions.getSecond().getRegionName(), tracker, family, mutations);
}
- stats.deleteStats(mergeRegions.getSecond().getRegionName(), tracker, family, mutations);
}
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Deleting the stats for the region " + region.getRegionNameAsString()
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Deleting the stats for the region " + region.getRegionNameAsString()
+ " as part of major compaction");
- }
- stats.deleteStats(region.getRegionName(), this.tracker, family, mutations);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Adding new stats for the region " + region.getRegionNameAsString()
+ }
+ stats.deleteStats(region.getRegionName(), tracker, family, mutations);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding new stats for the region " + region.getRegionNameAsString()
+ " as part of major compaction");
- }
- stats.addStats(region.getRegionName(), this.tracker, family, mutations);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Committing new stats for the region " + region.getRegionNameAsString()
+ }
+ stats.addStats(region.getRegionName(), tracker, family, mutations);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Committing new stats for the region " + region.getRegionNameAsString()
+ " as part of major compaction");
- }
- stats.commitStats(mutations);
- } catch (IOException e) {
- LOG.error("Failed to update statistics table!", e);
- toThrow = e;
- } finally {
- try {
- stats.close();
+ }
+ stats.commitStats(mutations);
} catch (IOException e) {
- if (toThrow == null) toThrow = e;
- LOG.error("Error while closing the stats table", e);
+ LOG.error("Failed to update statistics table!", e);
+ toThrow = e;
} finally {
- // close the delegate scanner
try {
- delegate.close();
+ statsRunState.removeCompactingRegion(region.getRegionInfo());
+ stats.close();
} catch (IOException e) {
if (toThrow == null) toThrow = e;
- LOG.error("Error while closing the scanner", e);
+ LOG.error("Error while closing the stats table", e);
} finally {
- if (toThrow != null) { throw toThrow; }
+ // close the delegate scanner
+ try {
+ delegate.close();
+ } catch (IOException e) {
+ if (toThrow == null) toThrow = e;
+ LOG.error("Error while closing the scanner", e);
+ } finally {
+ if (toThrow != null) { throw toThrow; }
+ }
}
}
+ return null;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ boolean async = config.getBoolean(COMMIT_STATS_ASYNC, DEFAULT_COMMIT_STATS_ASYNC);
+ StatisticsScannerCallable callable = new StatisticsScannerCallable();
+ if (!async) {
+ callable.call();
+ } else {
+ StatisticsCollectionRunTracker.getInstance(config).runTask(callable);
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/08aa07f5/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
index 8756568..3f187bd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
@@ -24,11 +24,8 @@ import java.util.Collections;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HTableInterface;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/08aa07f5/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
index 5289ab9..215110c 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
@@ -55,6 +55,8 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
public static final int DEFAULT_MIN_STATS_UPDATE_FREQ_MS = 0;
public static final boolean DEFAULT_EXPLAIN_CHUNK_COUNT = false; // TODO: update explain plans in test and set to true
public static final String DEFAULT_EXTRA_JDBC_ARGUMENTS = PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PARAM;
+ private static final boolean DEFAULT_RUN_UPDATE_STATS_ASYNC = false;
+ private static final boolean DEFAULT_COMMIT_STATS_ASYNC = false;
/**
@@ -93,7 +95,9 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
.setMaxClientMetaDataCacheSize(DEFAULT_MAX_CLIENT_METADATA_CACHE_SIZE)
.setMaxServerMetaDataCacheSize(DEFAULT_MAX_SERVER_METADATA_CACHE_SIZE)
.setForceRowKeyOrder(DEFAULT_FORCE_ROWKEY_ORDER)
- .setExtraJDBCArguments(DEFAULT_EXTRA_JDBC_ARGUMENTS);
+ .setExtraJDBCArguments(DEFAULT_EXTRA_JDBC_ARGUMENTS)
+ .setRunUpdateStatsAsync(DEFAULT_RUN_UPDATE_STATS_ASYNC)
+ .setCommitStatsAsync(DEFAULT_COMMIT_STATS_ASYNC);
}
public QueryServicesTestImpl(ReadOnlyProps defaultProps, ReadOnlyProps overrideProps) {
[02/50] [abbrv] phoenix git commit: PHOENIX-2515 Fix error if Hadoop
not installed
Posted by sa...@apache.org.
PHOENIX-2515 Fix error if Hadoop not installed
Don't log an error in bin scripts when looking for the
Hadoop classpath if Hadoop isn't installed.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/be0ccb4c
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/be0ccb4c
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/be0ccb4c
Branch: refs/heads/4.x-HBase-1.0
Commit: be0ccb4c0de9fc3bde18afce3f06f1f6eb664057
Parents: f157906
Author: Gabriel Reid <ga...@ngdata.com>
Authored: Fri Dec 11 13:50:47 2015 +0100
Committer: Gabriel Reid <gr...@apache.org>
Committed: Sun Dec 13 07:05:19 2015 +0100
----------------------------------------------------------------------
bin/phoenix_utils.py | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/be0ccb4c/bin/phoenix_utils.py
----------------------------------------------------------------------
diff --git a/bin/phoenix_utils.py b/bin/phoenix_utils.py
index 5fb4f08..6b00d5f 100755
--- a/bin/phoenix_utils.py
+++ b/bin/phoenix_utils.py
@@ -53,15 +53,18 @@ def findFileInPathWithoutRecursion(pattern, path):
return ""
-def which(file):
+def which(command):
for path in os.environ["PATH"].split(os.pathsep):
- if os.path.exists(os.path.join(path, file)):
- return os.path.join(path, file)
+ if os.path.exists(os.path.join(path, command)):
+ return os.path.join(path, command)
return None
-def findClasspath(file):
- aPath = which(file)
- command = "%s%s" %(aPath, ' classpath')
+def findClasspath(command_name):
+ command_path = which(command_name)
+ if command_path is None:
+ # We don't have this command, so we can't get its classpath
+ return ''
+ command = "%s%s" %(command_path, ' classpath')
return subprocess.Popen(command, shell=True, stdout=subprocess.PIPE).stdout.read()
def setPath():
[28/50] [abbrv] phoenix git commit: PHOENIX-2453 Make prepared
statement creation delegate to
org.apache.phoenix.jdbc.PhoenixConnection#prepareStatement(java.lang.String)
(Clement Escoffier)
Posted by sa...@apache.org.
PHOENIX-2453 Make prepared statement creation delegate to org.apache.phoenix.jdbc.PhoenixConnection#prepareStatement(java.lang.String) (Clement Escoffier)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ca101714
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ca101714
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ca101714
Branch: refs/heads/4.x-HBase-1.0
Commit: ca101714a60ff01450abeb8bfe107c4926295c53
Parents: cd4f454
Author: James Taylor <jt...@salesforce.com>
Authored: Mon Dec 28 22:42:46 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Dec 30 18:14:25 2015 -0800
----------------------------------------------------------------------
.../apache/phoenix/jdbc/PhoenixConnection.java | 9 ++++--
.../apache/phoenix/jdbc/PhoenixStatement.java | 12 ++++----
.../jdbc/PhoenixPreparedStatementTest.java | 30 +++++++++++++++++---
3 files changed, 38 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ca101714/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
index d33f20c..6a25184 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
@@ -701,17 +701,20 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
@Override
public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException {
- throw new SQLFeatureNotSupportedException();
+ // Ignore autoGeneratedKeys, and just execute the statement.
+ return prepareStatement(sql);
}
@Override
public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException {
- throw new SQLFeatureNotSupportedException();
+ // Ignore columnIndexes, and just execute the statement.
+ return prepareStatement(sql);
}
@Override
public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException {
- throw new SQLFeatureNotSupportedException();
+ // Ignore columnNames, and just execute the statement.
+ return prepareStatement(sql);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ca101714/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index fe62ea4..cb44c3f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -1352,32 +1352,32 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho
@Override
public boolean execute(String sql, int autoGeneratedKeys) throws SQLException {
- throw new SQLFeatureNotSupportedException();
+ return execute(sql);
}
@Override
public boolean execute(String sql, int[] columnIndexes) throws SQLException {
- throw new SQLFeatureNotSupportedException();
+ return execute(sql);
}
@Override
public boolean execute(String sql, String[] columnNames) throws SQLException {
- throw new SQLFeatureNotSupportedException();
+ return execute(sql);
}
@Override
public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException {
- throw new SQLFeatureNotSupportedException();
+ return executeUpdate(sql);
}
@Override
public int executeUpdate(String sql, int[] columnIndexes) throws SQLException {
- throw new SQLFeatureNotSupportedException();
+ return executeUpdate(sql);
}
@Override
public int executeUpdate(String sql, String[] columnNames) throws SQLException {
- throw new SQLFeatureNotSupportedException();
+ return executeUpdate(sql);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ca101714/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixPreparedStatementTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixPreparedStatementTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixPreparedStatementTest.java
index f9b0274..287fb4b 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixPreparedStatementTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixPreparedStatementTest.java
@@ -20,14 +20,12 @@ package org.apache.phoenix.jdbc;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
+import java.sql.*;
import java.util.Properties;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.query.BaseConnectionlessQueryTest;
+import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.junit.Test;
@@ -180,4 +178,28 @@ public class PhoenixPreparedStatementTest extends BaseConnectionlessQueryTest {
assertEquals(0, phoenixStmt.getQueryTimeoutInMillis());
}
+ @Test
+ public void testPreparedStatementWithGeneratedKeys() throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ PreparedStatement statement = conn.prepareStatement("CREATE TABLE T (pk1 CHAR(15) not null, pk2 VARCHAR not null, v1 VARCHAR(15), v2 DATE, " +
+ "v3 VARCHAR CONSTRAINT pk PRIMARY KEY (pk1, pk2))", PreparedStatement.RETURN_GENERATED_KEYS);
+ statement.execute();
+ }
+
+ @Test
+ public void testPreparedStatementWithColumnIndexes() throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ PreparedStatement statement = conn.prepareStatement("CREATE TABLE T (pk1 CHAR(15) not null, pk2 VARCHAR not null, v1 VARCHAR(15), v2 DATE, " +
+ "v3 VARCHAR CONSTRAINT pk PRIMARY KEY (pk1, pk2))", new int[0]);
+ statement.execute();
+ }
+
+ @Test
+ public void testPreparedStatementWithColumnNames() throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ PreparedStatement statement = conn.prepareStatement("CREATE TABLE T (pk1 CHAR(15) not null, pk2 VARCHAR not null, v1 VARCHAR(15), v2 DATE, " +
+ "v3 VARCHAR CONSTRAINT pk PRIMARY KEY (pk1, pk2))", new String[0]);
+ statement.execute();
+ }
+
}
[23/50] [abbrv] phoenix git commit: PHOENIX-2525 Views on
transactional tables return no rows
Posted by sa...@apache.org.
PHOENIX-2525 Views on transactional tables return no rows
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/276069a4
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/276069a4
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/276069a4
Branch: refs/heads/4.x-HBase-1.0
Commit: 276069a45ae7b1564e632d9f9ea1ffc6647b8a20
Parents: 8b1a732
Author: Thomas D'Silva <td...@salesforce.com>
Authored: Tue Dec 15 12:21:59 2015 -0800
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Wed Dec 23 14:52:17 2015 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/BaseViewIT.java | 62 ++++++--
.../apache/phoenix/end2end/SaltedViewIT.java | 8 +-
.../java/org/apache/phoenix/end2end/ViewIT.java | 145 +++++++++++--------
.../org/apache/phoenix/rpc/UpdateCacheIT.java | 2 +-
.../org/apache/phoenix/tx/TransactionIT.java | 31 ++++
.../phoenix/compile/CreateTableCompiler.java | 4 +-
.../phoenix/exception/SQLExceptionCode.java | 2 +-
.../apache/phoenix/execute/MutationState.java | 2 +
.../apache/phoenix/schema/MetaDataClient.java | 13 +-
.../java/org/apache/phoenix/util/TestUtil.java | 6 +
10 files changed, 189 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/276069a4/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
index 3140077..c713df4 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
@@ -27,6 +27,8 @@ import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -34,21 +36,50 @@ import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
import org.junit.BeforeClass;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
import com.google.common.collect.Maps;
-
+@RunWith(Parameterized.class)
public abstract class BaseViewIT extends BaseOwnClusterHBaseManagedTimeIT {
+
+ protected String tableName;
+ protected String fullTableName;
+ protected String tableDDLOptions;
+ protected String tableSuffix;
+ protected boolean transactional;
@BeforeClass
public static void doSetup() throws Exception {
- Map<String,String> props = Maps.newHashMapWithExpectedSize(1);
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
props.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Integer.toString(20));
props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(1024));
+ props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
+ public BaseViewIT( boolean transactional) {
+ StringBuilder optionBuilder = new StringBuilder();
+ this.transactional = transactional;
+ if (transactional) {
+ optionBuilder.append(" TRANSACTIONAL=true ");
+ }
+ this.tableDDLOptions = optionBuilder.toString();
+ tableSuffix = transactional ? "_TXN" : "";
+ this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + tableSuffix;
+ this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ }
+
+ @Parameters(name="transactional = {0}")
+ public static Collection<Boolean> data() {
+ return Arrays.asList(new Boolean[] { false, true });
+ }
+
protected void testUpdatableViewWithIndex(Integer saltBuckets, boolean localIndex) throws Exception {
testUpdatableView(saltBuckets);
testUpdatableViewIndex(saltBuckets, localIndex);
@@ -56,17 +87,22 @@ public abstract class BaseViewIT extends BaseOwnClusterHBaseManagedTimeIT {
protected void testUpdatableView(Integer saltBuckets) throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE t (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, k3 DECIMAL, s VARCHAR CONSTRAINT pk PRIMARY KEY (k1, k2, k3))" + (saltBuckets == null ? "" : (" SALT_BUCKETS="+saltBuckets));
+ if (saltBuckets!=null) {
+ if (tableDDLOptions.length()!=0)
+ tableDDLOptions+=",";
+ tableDDLOptions+=(" SALT_BUCKETS="+saltBuckets);
+ }
+ String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, k3 DECIMAL, s VARCHAR CONSTRAINT pk PRIMARY KEY (k1, k2, k3))" + tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW v AS SELECT * FROM t WHERE k1 = 1";
+ ddl = "CREATE VIEW v AS SELECT * FROM " + fullTableName + " WHERE k1 = 1";
conn.createStatement().execute(ddl);
for (int i = 0; i < 10; i++) {
- conn.createStatement().execute("UPSERT INTO t VALUES(" + (i % 4) + "," + (i+100) + "," + (i > 5 ? 2 : 1) + ")");
+ conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES(" + (i % 4) + "," + (i+100) + "," + (i > 5 ? 2 : 1) + ")");
}
conn.commit();
ResultSet rs;
- rs = conn.createStatement().executeQuery("SELECT count(*) FROM t");
+ rs = conn.createStatement().executeQuery("SELECT count(*) FROM " + fullTableName);
assertTrue(rs.next());
assertEquals(10, rs.getInt(1));
rs = conn.createStatement().executeQuery("SELECT count(*) FROM v");
@@ -132,14 +168,14 @@ public abstract class BaseViewIT extends BaseOwnClusterHBaseManagedTimeIT {
rs = conn.createStatement().executeQuery("EXPLAIN " + query);
String queryPlan = QueryUtil.getExplainPlan(rs);
if (localIndex) {
- assertEquals("CLIENT PARALLEL "+ (saltBuckets == null ? 1 : saltBuckets) +"-WAY RANGE SCAN OVER _LOCAL_IDX_T [-32768,51]\n"
+ assertEquals("CLIENT PARALLEL "+ (saltBuckets == null ? 1 : saltBuckets) +"-WAY RANGE SCAN OVER _LOCAL_IDX_" + tableName +" [-32768,51]\n"
+ " SERVER FILTER BY FIRST KEY ONLY\n"
+ "CLIENT MERGE SORT",
queryPlan);
} else {
assertEquals(saltBuckets == null
- ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _IDX_T [" + Short.MIN_VALUE + ",51]"
- : "CLIENT PARALLEL " + saltBuckets + "-WAY RANGE SCAN OVER _IDX_T [0," + Short.MIN_VALUE + ",51]\nCLIENT MERGE SORT",
+ ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _IDX_" + tableName +" [" + Short.MIN_VALUE + ",51]"
+ : "CLIENT PARALLEL " + saltBuckets + "-WAY RANGE SCAN OVER _IDX_T" + (transactional ? "_TXN" : "") + " [0," + Short.MIN_VALUE + ",51]\nCLIENT MERGE SORT",
queryPlan);
}
@@ -154,7 +190,7 @@ public abstract class BaseViewIT extends BaseOwnClusterHBaseManagedTimeIT {
assertEquals(saltBuckets == null ? 1 : 3, splits.size());
// analyze table should analyze all view data
- analyzeTable(conn, "t");
+ analyzeTable(conn, tableName);
splits = getAllSplits(conn, "i2");
assertEquals(saltBuckets == null ? 6 : 8, splits.size());
@@ -168,14 +204,14 @@ public abstract class BaseViewIT extends BaseOwnClusterHBaseManagedTimeIT {
assertFalse(rs.next());
rs = conn.createStatement().executeQuery("EXPLAIN " + query);
if (localIndex) {
- assertEquals("CLIENT PARALLEL "+ (saltBuckets == null ? 1 : saltBuckets) +"-WAY RANGE SCAN OVER _LOCAL_IDX_T [" + (Short.MIN_VALUE+1) + ",'foo']\n"
+ assertEquals("CLIENT PARALLEL "+ (saltBuckets == null ? 1 : saltBuckets) +"-WAY RANGE SCAN OVER _LOCAL_IDX_" + tableName +" [" + (Short.MIN_VALUE+1) + ",'foo']\n"
+ " SERVER FILTER BY FIRST KEY ONLY\n"
+ "CLIENT MERGE SORT",QueryUtil.getExplainPlan(rs));
} else {
assertEquals(saltBuckets == null
- ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _IDX_T [" + (Short.MIN_VALUE+1) + ",'foo']\n"
+ ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _IDX_" + tableName +" [" + (Short.MIN_VALUE+1) + ",'foo']\n"
+ " SERVER FILTER BY FIRST KEY ONLY"
- : "CLIENT PARALLEL " + saltBuckets + "-WAY RANGE SCAN OVER _IDX_T [0," + (Short.MIN_VALUE+1) + ",'foo']\n"
+ : "CLIENT PARALLEL " + saltBuckets + "-WAY RANGE SCAN OVER _IDX_T" + (transactional ? "_TXN" : "") + " [0," + (Short.MIN_VALUE+1) + ",'foo']\n"
+ " SERVER FILTER BY FIRST KEY ONLY\n"
+ "CLIENT MERGE SORT",
QueryUtil.getExplainPlan(rs));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/276069a4/phoenix-core/src/it/java/org/apache/phoenix/end2end/SaltedViewIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SaltedViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SaltedViewIT.java
index 19f0a4a..e7518f6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SaltedViewIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SaltedViewIT.java
@@ -21,8 +21,12 @@ import org.junit.Test;
public class SaltedViewIT extends BaseViewIT {
-
- /**
+
+ public SaltedViewIT(boolean transactional) {
+ super(transactional);
+ }
+
+ /**
* Salted tests must be in their own test file to ensure that the underlying
* table is dropped. Otherwise, the splits may not be performed.
* TODO: we should throw in that case
http://git-wip-us.apache.org/repos/asf/phoenix/blob/276069a4/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java
index 123fd85..55f26cd 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java
@@ -47,14 +47,18 @@ import org.junit.Test;
public class ViewIT extends BaseViewIT {
-
- @Test
+
+ public ViewIT(boolean transactional) {
+ super(transactional);
+ }
+
+ @Test
public void testReadOnlyView() throws Exception {
Connection earlierCon = DriverManager.getConnection(getUrl());
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE t (k INTEGER NOT NULL PRIMARY KEY, v1 DATE)";
+ String ddl = "CREATE TABLE " + fullTableName + " (k INTEGER NOT NULL PRIMARY KEY, v1 DATE) "+ tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW v (v2 VARCHAR) AS SELECT * FROM t WHERE k > 5";
+ ddl = "CREATE VIEW v (v2 VARCHAR) AS SELECT * FROM " + tableName + " WHERE k > 5";
conn.createStatement().execute(ddl);
try {
conn.createStatement().execute("UPSERT INTO v VALUES(1)");
@@ -63,17 +67,24 @@ public class ViewIT extends BaseViewIT {
}
for (int i = 0; i < 10; i++) {
- conn.createStatement().execute("UPSERT INTO t VALUES(" + i + ")");
+ conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES(" + i + ")");
}
conn.commit();
- analyzeTable(conn, "v");
+ analyzeTable(conn, "v", transactional);
List<KeyRange> splits = getAllSplits(conn, "v");
assertEquals(4, splits.size());
int count = 0;
- ResultSet rs = conn.createStatement().executeQuery("SELECT k FROM v");
+ ResultSet rs = conn.createStatement().executeQuery("SELECT k FROM " + tableName);
+ while (rs.next()) {
+ assertEquals(count++, rs.getInt(1));
+ }
+ assertEquals(10, count);
+
+ count = 0;
+ rs = conn.createStatement().executeQuery("SELECT k FROM v");
while (rs.next()) {
count++;
assertEquals(count + 5, rs.getInt(1));
@@ -88,7 +99,6 @@ public class ViewIT extends BaseViewIT {
assertEquals(4, count);
}
-
@Test
public void testReadOnlyOnReadOnlyView() throws Exception {
testReadOnlyView();
@@ -181,7 +191,7 @@ public class ViewIT extends BaseViewIT {
}
- conn.createStatement().execute("UPSERT INTO t(k1, k2,k3) VALUES(1, 122, 5)");
+ conn.createStatement().execute("UPSERT INTO " + fullTableName + "(k1, k2,k3) VALUES(1, 122, 5)");
conn.commit();
rs = conn.createStatement().executeQuery("SELECT k1, k2, k3 FROM v2 WHERE k2 >= 120");
assertTrue(rs.next());
@@ -194,9 +204,9 @@ public class ViewIT extends BaseViewIT {
@Test
public void testDisallowDropOfReferencedColumn() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE t (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))";
+ String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))" + tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW v1(v2 VARCHAR, v3 VARCHAR) AS SELECT * FROM t WHERE v1 = 1.0";
+ ddl = "CREATE VIEW v1(v2 VARCHAR, v3 VARCHAR) AS SELECT * FROM " + fullTableName + " WHERE v1 = 1.0";
conn.createStatement().execute(ddl);
try {
@@ -229,9 +239,10 @@ public class ViewIT extends BaseViewIT {
public void testReadOnlyViewWithCaseSensitiveTableNames() throws Exception {
Connection earlierCon = DriverManager.getConnection(getUrl());
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE \"case_SENSITIVE_table\" (k INTEGER NOT NULL PRIMARY KEY, v1 DATE)";
+ String caseSensitiveTableName = "\"case_SENSITIVE_table" + tableSuffix + "\"" ;
+ String ddl = "CREATE TABLE " + caseSensitiveTableName + " (k INTEGER NOT NULL PRIMARY KEY, v1 DATE)" + tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW \"v\" (v2 VARCHAR) AS SELECT * FROM \"case_SENSITIVE_table\" WHERE k > 5";
+ ddl = "CREATE VIEW \"v\" (v2 VARCHAR) AS SELECT * FROM " + caseSensitiveTableName + " WHERE k > 5";
conn.createStatement().execute(ddl);
try {
conn.createStatement().execute("UPSERT INTO \"v\" VALUES(1)");
@@ -240,7 +251,7 @@ public class ViewIT extends BaseViewIT {
}
for (int i = 0; i < 10; i++) {
- conn.createStatement().execute("UPSERT INTO \"case_SENSITIVE_table\" VALUES(" + i + ")");
+ conn.createStatement().execute("UPSERT INTO " + caseSensitiveTableName + " VALUES(" + i + ")");
}
conn.commit();
@@ -263,9 +274,9 @@ public class ViewIT extends BaseViewIT {
@Test
public void testReadOnlyViewWithCaseSensitiveColumnNames() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE t (\"k\" INTEGER NOT NULL PRIMARY KEY, \"v1\" INTEGER, \"a\".v2 VARCHAR)";
+ String ddl = "CREATE TABLE " + fullTableName + " (\"k\" INTEGER NOT NULL PRIMARY KEY, \"v1\" INTEGER, \"a\".v2 VARCHAR)" + tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW v (v VARCHAR) AS SELECT * FROM t WHERE \"k\" > 5 and \"v1\" > 1";
+ ddl = "CREATE VIEW v (v VARCHAR) AS SELECT * FROM " + fullTableName + " WHERE \"k\" > 5 and \"v1\" > 1";
conn.createStatement().execute(ddl);
try {
conn.createStatement().execute("UPSERT INTO v VALUES(1)");
@@ -274,7 +285,7 @@ public class ViewIT extends BaseViewIT {
}
for (int i = 0; i < 10; i++) {
- conn.createStatement().execute("UPSERT INTO t VALUES(" + i + ", " + (i+10) + ",'A')");
+ conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES(" + i + ", " + (i+10) + ",'A')");
}
conn.commit();
@@ -290,9 +301,9 @@ public class ViewIT extends BaseViewIT {
@Test
public void testViewWithCurrentDate() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE t (k INTEGER NOT NULL PRIMARY KEY, v1 INTEGER, v2 DATE)";
+ String ddl = "CREATE TABLE " + fullTableName + " (k INTEGER NOT NULL PRIMARY KEY, v1 INTEGER, v2 DATE)" + tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW v (v VARCHAR) AS SELECT * FROM t WHERE v2 > CURRENT_DATE()-5 AND v2 > DATE '2010-01-01'";
+ ddl = "CREATE VIEW v (v VARCHAR) AS SELECT * FROM " + fullTableName + " WHERE v2 > CURRENT_DATE()-5 AND v2 > DATE '2010-01-01'";
conn.createStatement().execute(ddl);
try {
conn.createStatement().execute("UPSERT INTO v VALUES(1)");
@@ -301,7 +312,7 @@ public class ViewIT extends BaseViewIT {
}
for (int i = 0; i < 10; i++) {
- conn.createStatement().execute("UPSERT INTO t VALUES(" + i + ", " + (i+10) + ",CURRENT_DATE()-" + i + ")");
+ conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES(" + i + ", " + (i+10) + ",CURRENT_DATE()-" + i + ")");
}
conn.commit();
@@ -317,11 +328,12 @@ public class ViewIT extends BaseViewIT {
@Test
public void testViewAndTableInDifferentSchemas() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE s1.t (k INTEGER NOT NULL PRIMARY KEY, v1 DATE)";
+ String fullTableName = "s1.t"+tableSuffix;
+ String ddl = "CREATE TABLE " + fullTableName + " (k INTEGER NOT NULL PRIMARY KEY, v1 DATE)" + tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW s2.v1 (v2 VARCHAR) AS SELECT * FROM s1.t WHERE k > 5";
+ ddl = "CREATE VIEW s2.v1 (v2 VARCHAR) AS SELECT * FROM " + fullTableName + " WHERE k > 5";
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW v2 (v2 VARCHAR) AS SELECT * FROM s1.t WHERE k > 5";
+ ddl = "CREATE VIEW v2 (v2 VARCHAR) AS SELECT * FROM " + fullTableName + " WHERE k > 5";
conn.createStatement().execute(ddl);
ddl = "DROP VIEW v1";
try {
@@ -337,11 +349,11 @@ public class ViewIT extends BaseViewIT {
fail();
} catch (TableNotFoundException ignore) {
}
- ddl = "DROP TABLE s1.t";
- validateCannotDropTableWithChildViewsWithoutCascade(conn, "s1.t");
+ ddl = "DROP TABLE " + fullTableName;
+ validateCannotDropTableWithChildViewsWithoutCascade(conn, fullTableName);
ddl = "DROP VIEW v2";
conn.createStatement().execute(ddl);
- ddl = "DROP TABLE s1.t";
+ ddl = "DROP TABLE " + fullTableName;
conn.createStatement().execute(ddl);
}
@@ -349,13 +361,13 @@ public class ViewIT extends BaseViewIT {
@Test
public void testDisallowDropOfColumnOnParentTable() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE tp (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))";
+ String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))" + tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW v1(v2 VARCHAR, v3 VARCHAR) AS SELECT * FROM tp WHERE v1 = 1.0";
+ ddl = "CREATE VIEW v1(v2 VARCHAR, v3 VARCHAR) AS SELECT * FROM " + fullTableName + " WHERE v1 = 1.0";
conn.createStatement().execute(ddl);
try {
- conn.createStatement().execute("ALTER TABLE tp DROP COLUMN v1");
+ conn.createStatement().execute("ALTER TABLE " + fullTableName + " DROP COLUMN v1");
fail();
} catch (SQLException e) {
assertEquals(SQLExceptionCode.CANNOT_MUTATE_TABLE.getErrorCode(), e.getErrorCode());
@@ -365,18 +377,19 @@ public class ViewIT extends BaseViewIT {
@Test
public void testViewAndTableAndDropCascade() throws Exception {
// Setup
+ String fullTableName = "s2.t"+tableSuffix;
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE s2.t (k INTEGER NOT NULL PRIMARY KEY, v1 DATE)";
+ String ddl = "CREATE TABLE " + fullTableName + " (k INTEGER NOT NULL PRIMARY KEY, v1 DATE)" + tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW s2.v1 (v2 VARCHAR) AS SELECT * FROM s2.t WHERE k > 5";
+ ddl = "CREATE VIEW s2.v1 (v2 VARCHAR) AS SELECT * FROM " + fullTableName + " WHERE k > 5";
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW s2.v2 (v2 VARCHAR) AS SELECT * FROM s2.t WHERE k > 10";
+ ddl = "CREATE VIEW s2.v2 (v2 VARCHAR) AS SELECT * FROM " + fullTableName + " WHERE k > 10";
conn.createStatement().execute(ddl);
- validateCannotDropTableWithChildViewsWithoutCascade(conn, "s2.t");
+ validateCannotDropTableWithChildViewsWithoutCascade(conn, fullTableName);
// Execute DROP...CASCADE
- conn.createStatement().execute("DROP TABLE s2.t CASCADE");
+ conn.createStatement().execute("DROP TABLE " + fullTableName + " CASCADE");
validateViewDoesNotExist(conn, "s2.v1");
validateViewDoesNotExist(conn, "s2.v2");
@@ -387,24 +400,27 @@ public class ViewIT extends BaseViewIT {
// Setup - Tables and Views with Indexes
Connection conn = DriverManager.getConnection(getUrl());
-
- String ddl = "CREATE TABLE s3.t (k INTEGER NOT NULL PRIMARY KEY, v1 DATE) IMMUTABLE_ROWS=true";
+ String fullTableName = "s3.t"+tableSuffix;
+ if (tableDDLOptions.length()!=0)
+ tableDDLOptions+=",";
+ tableDDLOptions+="IMMUTABLE_ROWS=true";
+ String ddl = "CREATE TABLE " + fullTableName + " (k INTEGER NOT NULL PRIMARY KEY, v1 DATE)" + tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE INDEX IDX1 ON s3.t (v1)";
+ ddl = "CREATE INDEX IDX1 ON " + fullTableName + " (v1)";
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW s3.v1 (v2 VARCHAR) AS SELECT * FROM s3.t WHERE k > 5";
+ ddl = "CREATE VIEW s3.v1 (v2 VARCHAR) AS SELECT * FROM " + fullTableName + " WHERE k > 5";
conn.createStatement().execute(ddl);
ddl = "CREATE INDEX IDX2 ON s3.v1 (v2)";
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW s3.v2 (v2 VARCHAR) AS SELECT * FROM s3.t WHERE k > 10";
+ ddl = "CREATE VIEW s3.v2 (v2 VARCHAR) AS SELECT * FROM " + fullTableName + " WHERE k > 10";
conn.createStatement().execute(ddl);
ddl = "CREATE INDEX IDX3 ON s3.v2 (v2)";
conn.createStatement().execute(ddl);
- validateCannotDropTableWithChildViewsWithoutCascade(conn, "s3.t");
+ validateCannotDropTableWithChildViewsWithoutCascade(conn, fullTableName);
// Execute DROP...CASCADE
- conn.createStatement().execute("DROP TABLE s3.t CASCADE");
+ conn.createStatement().execute("DROP TABLE " + fullTableName + " CASCADE");
// Validate Views were deleted - Try and delete child views, should throw TableNotFoundException
validateViewDoesNotExist(conn, "s3.v1");
@@ -438,16 +454,16 @@ public class ViewIT extends BaseViewIT {
public void testViewUsesTableIndex() throws Exception {
ResultSet rs;
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE t (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, k3 DECIMAL, s1 VARCHAR, s2 VARCHAR CONSTRAINT pk PRIMARY KEY (k1, k2, k3))";
+ String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, k3 DECIMAL, s1 VARCHAR, s2 VARCHAR CONSTRAINT pk PRIMARY KEY (k1, k2, k3))" + tableDDLOptions;
conn.createStatement().execute(ddl);
- conn.createStatement().execute("CREATE INDEX i1 ON t(k3, k2) INCLUDE(s1, s2)");
- conn.createStatement().execute("CREATE INDEX i2 ON t(k3, k2, s2)");
+ conn.createStatement().execute("CREATE INDEX i1 ON " + fullTableName + "(k3, k2) INCLUDE(s1, s2)");
+ conn.createStatement().execute("CREATE INDEX i2 ON " + fullTableName + "(k3, k2, s2)");
- ddl = "CREATE VIEW v AS SELECT * FROM t WHERE s1 = 'foo'";
+ ddl = "CREATE VIEW v AS SELECT * FROM " + fullTableName + " WHERE s1 = 'foo'";
conn.createStatement().execute(ddl);
String[] s1Values = {"foo","bar"};
for (int i = 0; i < 10; i++) {
- conn.createStatement().execute("UPSERT INTO t VALUES(" + (i % 4) + "," + (i+100) + "," + (i > 5 ? 2 : 1) + ",'" + s1Values[i%2] + "','bas')");
+ conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES(" + (i % 4) + "," + (i+100) + "," + (i > 5 ? 2 : 1) + ",'" + s1Values[i%2] + "','bas')");
}
conn.commit();
@@ -473,9 +489,9 @@ public class ViewIT extends BaseViewIT {
@Test
public void testCreateViewDefinesPKColumn() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE tp (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))";
+ String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))" + tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW v1(v2 VARCHAR, k3 VARCHAR PRIMARY KEY) AS SELECT * FROM tp WHERE K1 = 1";
+ ddl = "CREATE VIEW v1(v2 VARCHAR, k3 VARCHAR PRIMARY KEY) AS SELECT * FROM " + fullTableName + " WHERE K1 = 1";
conn.createStatement().execute(ddl);
// assert PK metadata
@@ -483,12 +499,12 @@ public class ViewIT extends BaseViewIT {
assertPKs(rs, new String[] {"K1", "K2", "K3"});
// sanity check upserts into base table and view
- conn.createStatement().executeUpdate("upsert into tp (k1, k2, v1) values (1, 1, 1)");
+ conn.createStatement().executeUpdate("upsert into " + fullTableName + " (k1, k2, v1) values (1, 1, 1)");
conn.createStatement().executeUpdate("upsert into v1 (k1, k2, k3, v2) values (1, 1, 'abc', 'def')");
conn.commit();
// expect 2 rows in the base table
- rs = conn.createStatement().executeQuery("select count(*) from tp");
+ rs = conn.createStatement().executeQuery("select count(*) from " + fullTableName);
assertTrue(rs.next());
assertEquals(2, rs.getInt(1));
@@ -501,9 +517,9 @@ public class ViewIT extends BaseViewIT {
@Test
public void testCreateViewDefinesPKConstraint() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE tp (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))";
+ String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))" + tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW v1(v2 VARCHAR, k3 VARCHAR, k4 INTEGER NOT NULL, CONSTRAINT PKVEW PRIMARY KEY (k3, k4)) AS SELECT * FROM tp WHERE K1 = 1";
+ ddl = "CREATE VIEW v1(v2 VARCHAR, k3 VARCHAR, k4 INTEGER NOT NULL, CONSTRAINT PKVEW PRIMARY KEY (k3, k4)) AS SELECT * FROM " + fullTableName + " WHERE K1 = 1";
conn.createStatement().execute(ddl);
// assert PK metadata
@@ -514,9 +530,10 @@ public class ViewIT extends BaseViewIT {
@Test
public void testViewAddsPKColumn() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE tp (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))";
+ String fullTableName2 = fullTableName;
+ String ddl = "CREATE TABLE " + fullTableName2 + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))" + tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW v1 AS SELECT * FROM tp WHERE v1 = 1.0";
+ ddl = "CREATE VIEW v1 AS SELECT * FROM " + fullTableName + " WHERE v1 = 1.0";
conn.createStatement().execute(ddl);
ddl = "ALTER VIEW V1 ADD k3 VARCHAR PRIMARY KEY, k4 VARCHAR PRIMARY KEY, v2 INTEGER";
conn.createStatement().execute(ddl);
@@ -529,9 +546,9 @@ public class ViewIT extends BaseViewIT {
@Test
public void testViewAddsPKColumnWhoseParentsLastPKIsVarLength() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE tp (k1 INTEGER NOT NULL, k2 VARCHAR NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))";
+ String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 VARCHAR NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))" + tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW v1 AS SELECT * FROM tp WHERE v1 = 1.0";
+ ddl = "CREATE VIEW v1 AS SELECT * FROM " + fullTableName + " WHERE v1 = 1.0";
conn.createStatement().execute(ddl);
ddl = "ALTER VIEW V1 ADD k3 VARCHAR PRIMARY KEY, k4 VARCHAR PRIMARY KEY, v2 INTEGER";
try {
@@ -540,7 +557,7 @@ public class ViewIT extends BaseViewIT {
} catch (SQLException e) {
assertEquals(CANNOT_MODIFY_VIEW_PK.getErrorCode(), e.getErrorCode());
}
- ddl = "CREATE VIEW v2 (k3 VARCHAR PRIMARY KEY) AS SELECT * FROM tp WHERE v1 = 1.0";
+ ddl = "CREATE VIEW v2 (k3 VARCHAR PRIMARY KEY) AS SELECT * FROM " + fullTableName + " WHERE v1 = 1.0";
try {
conn.createStatement().execute(ddl);
} catch (SQLException e) {
@@ -551,9 +568,9 @@ public class ViewIT extends BaseViewIT {
@Test(expected=ColumnAlreadyExistsException.class)
public void testViewAddsClashingPKColumn() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE tp (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))";
+ String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))" + tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW v1 AS SELECT * FROM tp WHERE v1 = 1.0";
+ ddl = "CREATE VIEW v1 AS SELECT * FROM " + fullTableName + " WHERE v1 = 1.0";
conn.createStatement().execute(ddl);
ddl = "ALTER VIEW V1 ADD k3 VARCHAR PRIMARY KEY, k2 VARCHAR PRIMARY KEY, v2 INTEGER";
conn.createStatement().execute(ddl);
@@ -562,9 +579,9 @@ public class ViewIT extends BaseViewIT {
@Test
public void testViewAddsNotNullPKColumn() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
- String ddl = "CREATE TABLE tp (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))";
+ String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))" + tableDDLOptions;
conn.createStatement().execute(ddl);
- ddl = "CREATE VIEW v1 AS SELECT * FROM tp WHERE v1 = 1.0";
+ ddl = "CREATE VIEW v1 AS SELECT * FROM " + fullTableName + " WHERE v1 = 1.0";
conn.createStatement().execute(ddl);
try {
ddl = "ALTER VIEW V1 ADD k3 VARCHAR NOT NULL PRIMARY KEY";
@@ -578,11 +595,11 @@ public class ViewIT extends BaseViewIT {
@Test
public void testQueryViewStatementOptimization() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
- String sql = "CREATE TABLE tp (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))";
+ String sql = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, v1 DECIMAL, CONSTRAINT pk PRIMARY KEY (k1, k2))" + tableDDLOptions;
conn.createStatement().execute(sql);
- sql = "CREATE VIEW v1 AS SELECT * FROM tp";
+ sql = "CREATE VIEW v1 AS SELECT * FROM " + fullTableName;
conn.createStatement().execute(sql);
- sql = "CREATE VIEW v2 AS SELECT * FROM tp WHERE k1 = 1.0";
+ sql = "CREATE VIEW v2 AS SELECT * FROM " + fullTableName + " WHERE k1 = 1.0";
conn.createStatement().execute(sql);
sql = "SELECT * FROM v1 order by k1, k2";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/276069a4/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
index 208af94..13ed8aa 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
@@ -168,7 +168,7 @@ public class UpdateCacheIT extends BaseHBaseManagedTimeIT {
// for non-transactional tables without a scn : verify one rpc to getTable occurs *per* query
// for non-transactional tables with a scn : verify *only* one rpc occurs
// for transactional tables : verify *only* one rpc occurs
- // for non-transactional, system tables : verify non rpc occurs
+ // for non-transactional, system tables : verify no rpc occurs
int numRpcs = isSystem ? 0 : (isTransactional || scn!=null ? 1 : 3);
verify(connectionQueryServices, times(numRpcs)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)), anyLong(), anyLong());
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/276069a4/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
index 70e31b9..929e4a5 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
@@ -20,6 +20,8 @@ package org.apache.phoenix.tx;
import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.apache.phoenix.util.TestUtil.TRANSACTIONAL_DATA_TABLE;
+import static org.apache.phoenix.util.TestUtil.analyzeTable;
+import static org.apache.phoenix.util.TestUtil.getAllSplits;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -48,11 +50,13 @@ import org.apache.phoenix.end2end.Shadower;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.ReadOnlyTableException;
import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.PropertiesUtil;
@@ -572,4 +576,31 @@ public class TransactionIT extends BaseHBaseManagedTimeIT {
}
}
}
+
+ @Test
+ public void testReadOnlyView() throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ String ddl = "CREATE TABLE t (k INTEGER NOT NULL PRIMARY KEY, v1 DATE) TRANSACTIONAL=true";
+ conn.createStatement().execute(ddl);
+ ddl = "CREATE VIEW v (v2 VARCHAR) AS SELECT * FROM t where k>4";
+ conn.createStatement().execute(ddl);
+ for (int i = 0; i < 10; i++) {
+ conn.createStatement().execute("UPSERT INTO t VALUES(" + i + ")");
+ }
+ conn.commit();
+
+ int count = 0;
+ ResultSet rs = conn.createStatement().executeQuery("SELECT k FROM t");
+ while (rs.next()) {
+ assertEquals(count++, rs.getInt(1));
+ }
+ assertEquals(10, count);
+
+ count = 0;
+ rs = conn.createStatement().executeQuery("SELECT k FROM v");
+ while (rs.next()) {
+ assertEquals(5+count++, rs.getInt(1));
+ }
+ assertEquals(5, count);
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/276069a4/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
index d5b74fb..e032feb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
@@ -120,8 +120,8 @@ public class CreateTableCompiler {
}
if (viewTypeToBe != ViewType.MAPPED) {
Long scn = connection.getSCN();
- connectionToBe = scn != null ? connection :
- // If we haved no SCN on our connection, freeze the SCN at when
+ connectionToBe = (scn != null || tableRef.getTable().isTransactional()) ? connection :
+ // If we haved no SCN on our connection and the base table is not transactional, freeze the SCN at when
// the base table was resolved to prevent any race condition on
// the error checking we do for the base table. The only potential
// issue is if the base table lives on a different region server
http://git-wip-us.apache.org/repos/asf/phoenix/blob/276069a4/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index 39f2c14..ef135eb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -274,7 +274,7 @@ public enum SQLExceptionCode {
CANNOT_START_TRANSACTION_WITH_SCN_SET(1073, "44A04", "Cannot start a transaction on a connection with SCN set"),
TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE(1074, "44A05", "A transactional table must define VERSION of greater than one"),
CANNOT_SPECIFY_SCN_FOR_TXN_TABLE(1075, "44A06", "Cannot use a connection with SCN set for a transactional table"),
- NULL_TRANSACTION_CONTEXT(1076, "44A07", "No Tranasction Context available"),
+ NULL_TRANSACTION_CONTEXT(1076, "44A07", "No Transaction Context available"),
TRANSACTION_FAILED(1077, "44A08", "Transaction Failure "),
CANNOT_CREATE_TXN_TABLE_IF_TXNS_DISABLED(1078, "44A09", "Cannot create a transactional table if transactions are disabled"),
CANNOT_ALTER_TO_BE_TXN_IF_TXNS_DISABLED(1079, "44A10", "Cannot alter table to be transactional table if transactions are disabled"),
http://git-wip-us.apache.org/repos/asf/phoenix/blob/276069a4/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
index c67b222..52bbeb6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -162,6 +162,8 @@ public class MutationState implements SQLCloseable {
.getQueryServices().getTransactionSystemClient();
this.txContext = new TransactionContext(txServiceClient);
} else {
+ // this code path is only used while running child scans, we can't pass the txContext to child scans
+ // as it is not thread safe, so we use the tx member variable
txAwares = Lists.newArrayList();
txContext = null;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/276069a4/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 35d61ff..7191a7c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -938,7 +938,15 @@ public class MetaDataClient {
* since it may not represent a "real" table in the case of the view indexes of a base table.
*/
PostDDLCompiler compiler = new PostDDLCompiler(connection);
- TableRef tableRef = new TableRef(null, logicalTable, clientTimeStamp, false);
+ //even if table is transactional, while calculating stats we scan the table non-transactionally to
+ //view all the data belonging to the table
+ PTable nonTxnLogicalTable = new DelegateTable(logicalTable) {
+ @Override
+ public boolean isTransactional() {
+ return false;
+ }
+ };
+ TableRef tableRef = new TableRef(null, nonTxnLogicalTable, clientTimeStamp, false);
MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, null, null, clientTimeStamp);
Scan scan = plan.getContext().getScan();
scan.setCacheBlocks(false);
@@ -1562,7 +1570,7 @@ public class MetaDataClient {
long clientTimeStamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
boolean multiTenant = false;
boolean storeNulls = false;
- boolean transactional = false;
+ boolean transactional = (parent!= null) ? parent.isTransactional() : false;
Integer saltBucketNum = null;
String defaultFamilyName = null;
boolean isImmutableRows = false;
@@ -1571,7 +1579,6 @@ public class MetaDataClient {
boolean rowKeyOrderOptimizable = true;
Long timestamp = null;
if (parent != null && tableType == PTableType.INDEX) {
- transactional = parent.isTransactional();
timestamp = TransactionUtil.getTableTimestamp(connection, transactional);
storeNulls = parent.getStoreNulls();
if (tableType == PTableType.INDEX) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/276069a4/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index e66f8ca..e1258e4 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -556,8 +556,14 @@ public class TestUtil {
}
public static void analyzeTable(Connection conn, String tableName) throws IOException, SQLException {
+ analyzeTable(conn, tableName, false);
+ }
+
+ public static void analyzeTable(Connection conn, String tableName, boolean transactional) throws IOException, SQLException {
String query = "UPDATE STATISTICS " + tableName;
conn.createStatement().execute(query);
+ // if the table is transactional burn a txn in order to make sure the next txn read pointer is close to wall clock time
+ TransactionUtil.getTableTimestamp(conn.unwrap(PhoenixConnection.class), transactional);
}
public static void analyzeTableIndex(Connection conn, String tableName) throws IOException, SQLException {
[47/50] [abbrv] phoenix git commit: PHOENIX-2569 Add missing ASL
header to IndexIT.java
Posted by sa...@apache.org.
PHOENIX-2569 Add missing ASL header to IndexIT.java
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7f555aaf
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7f555aaf
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7f555aaf
Branch: refs/heads/4.x-HBase-1.0
Commit: 7f555aafd65b7695455aef6e2387cc4c666fc7c7
Parents: ffe2490
Author: Josh Elser <el...@apache.org>
Authored: Tue Jan 5 14:05:47 2016 -0500
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Tue Jan 5 13:07:08 2016 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/index/IndexIT.java | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7f555aaf/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
index 72f2230..f48b847 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.phoenix.end2end.index;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
[33/50] [abbrv] phoenix git commit: PHOENIX-2552 Fix hanging Jenkins
builds
Posted by sa...@apache.org.
PHOENIX-2552 Fix hanging Jenkins builds
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/4c99fc2b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/4c99fc2b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/4c99fc2b
Branch: refs/heads/4.x-HBase-1.0
Commit: 4c99fc2b1c13b54820cbe0544980b392f6fcdf4b
Parents: fd5fff2
Author: James Taylor <jt...@salesforce.com>
Authored: Wed Dec 30 21:04:12 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Dec 30 23:17:56 2015 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/memory/GlobalMemoryManager.java | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4c99fc2b/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java b/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
index 6460260..e70b35f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
@@ -151,23 +151,24 @@ public class GlobalMemoryManager implements MemoryManager {
if (size > 0) {
logger.warn("Orphaned chunk of " + size + " bytes found during finalize");
}
- close();
- // TODO: log error here, but we can't use SFDC logging
- // because this runs in an hbase coprocessor.
- // Create a gack-like API (talk with GridForce or HBase folks)
+ freeMemory();
} finally {
super.finalize();
}
}
- @Override
- public void close() {
+ private void freeMemory() {
synchronized(sync) {
usedMemoryBytes -= size;
size = 0;
sync.notifyAll();
}
}
+
+ @Override
+ public void close() {
+ freeMemory();
+ }
}
}
[13/50] [abbrv] phoenix git commit: PHOENIX-2527
InsufficientMemoryException error message is misleading
Posted by sa...@apache.org.
PHOENIX-2527 InsufficientMemoryException error message is misleading
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/feec7c4f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/feec7c4f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/feec7c4f
Branch: refs/heads/4.x-HBase-1.0
Commit: feec7c4ffbdef515b6f553530e9d0c591156de6a
Parents: a2b40de
Author: Enis Soztutar <en...@apache.org>
Authored: Wed Dec 16 14:15:03 2015 -0800
Committer: Enis Soztutar <en...@apache.org>
Committed: Wed Dec 16 14:15:06 2015 -0800
----------------------------------------------------------------------
.../phoenix/memory/GlobalMemoryManager.java | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/feec7c4f/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java b/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
index 79b49c5..6460260 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
@@ -21,15 +21,15 @@ import org.apache.http.annotation.GuardedBy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- *
+ *
* Global memory manager to track course grained memory usage across all requests.
*
- *
+ *
* @since 0.1
*/
public class GlobalMemoryManager implements MemoryManager {
private static final Logger logger = LoggerFactory.getLogger(GlobalMemoryManager.class);
-
+
private final Object sync = new Object();
private final long maxMemoryBytes;
private final int maxWaitMs;
@@ -46,7 +46,7 @@ public class GlobalMemoryManager implements MemoryManager {
this.maxWaitMs = maxWaitMs;
this.usedMemoryBytes = 0;
}
-
+
@Override
public long getAvailableMemory() {
synchronized(sync) {
@@ -75,7 +75,7 @@ public class GlobalMemoryManager implements MemoryManager {
try {
long remainingWaitTimeMs = maxWaitMs - (System.currentTimeMillis() - startTimeMs);
if (remainingWaitTimeMs <= 0) { // Ran out of time waiting for some memory to get freed up
- throw new InsufficientMemoryException("Requested memory of " + minBytes + " bytes could not be allocated from remaining memory of " + usedMemoryBytes + " bytes from global pool of " + maxMemoryBytes + " bytes after waiting for " + maxWaitMs + "ms.");
+ throw new InsufficientMemoryException("Requested memory of " + minBytes + " bytes could not be allocated. Using memory of " + usedMemoryBytes + " bytes from global pool of " + maxMemoryBytes + " bytes after waiting for " + maxWaitMs + "ms.");
}
sync.wait(remainingWaitTimeMs);
} catch (InterruptedException ie) {
@@ -106,7 +106,7 @@ public class GlobalMemoryManager implements MemoryManager {
protected MemoryChunk newMemoryChunk(long sizeBytes) {
return new GlobalMemoryChunk(sizeBytes);
}
-
+
private class GlobalMemoryChunk implements MemoryChunk {
private volatile long size;
@@ -123,7 +123,7 @@ public class GlobalMemoryManager implements MemoryManager {
return size; // TODO: does this need to be synchronized?
}
}
-
+
@Override
public void resize(long nBytes) {
if (nBytes < 0) {
@@ -141,7 +141,7 @@ public class GlobalMemoryManager implements MemoryManager {
}
}
}
-
+
/**
* Check that MemoryChunk has previously been closed.
*/
@@ -159,7 +159,7 @@ public class GlobalMemoryManager implements MemoryManager {
super.finalize();
}
}
-
+
@Override
public void close() {
synchronized(sync) {
@@ -170,4 +170,4 @@ public class GlobalMemoryManager implements MemoryManager {
}
}
}
-
+
[36/50] [abbrv] phoenix git commit: PHOENIX-2555 Fix date test that
only works while it's 2015
Posted by sa...@apache.org.
PHOENIX-2555 Fix date test that only works while it's 2015
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/dd3028cd
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/dd3028cd
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/dd3028cd
Branch: refs/heads/4.x-HBase-1.0
Commit: dd3028cd0986aa43d0e7d486153c6a0ba2f82360
Parents: 58465b3
Author: James Taylor <jt...@salesforce.com>
Authored: Fri Jan 1 18:43:15 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Jan 1 18:44:43 2016 -0800
----------------------------------------------------------------------
.../src/it/java/org/apache/phoenix/end2end/DateTimeIT.java | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dd3028cd/phoenix-core/src/it/java/org/apache/phoenix/end2end/DateTimeIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DateTimeIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DateTimeIT.java
index b4876a5..0fce403 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DateTimeIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DateTimeIT.java
@@ -413,10 +413,6 @@ public class DateTimeIT extends BaseHBaseManagedTimeIT {
@Test
public void testYearFunctionDate() throws SQLException {
- assertEquals(2015, callYearFunction("YEAR(current_date())"));
-
- assertEquals(2015, callYearFunction("YEAR(now())"));
-
assertEquals(2008, callYearFunction("YEAR(TO_DATE('2008-01-01', 'yyyy-MM-dd', 'local'))"));
assertEquals(2004,
[40/50] [abbrv] phoenix git commit: PHOENIX-2560 PhoenixMetricsIT
test failures
Posted by sa...@apache.org.
PHOENIX-2560 PhoenixMetricsIT test failures
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/423bd134
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/423bd134
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/423bd134
Branch: refs/heads/4.x-HBase-1.0
Commit: 423bd134f6a9a25a2eaaa9d5d82b4be96cd6eba4
Parents: 1cd1d26
Author: James Taylor <jt...@salesforce.com>
Authored: Mon Jan 4 08:00:47 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Jan 4 09:36:46 2016 -0800
----------------------------------------------------------------------
.../MutatingParallelIteratorFactory.java | 18 +-------------
.../phoenix/iterate/BaseResultIterators.java | 3 +++
.../iterate/MergeSortResultIterator.java | 25 +-------------------
.../query/ConnectionQueryServicesImpl.java | 4 +++-
4 files changed, 8 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/423bd134/phoenix-core/src/main/java/org/apache/phoenix/compile/MutatingParallelIteratorFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/MutatingParallelIteratorFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/MutatingParallelIteratorFactory.java
index 630760c..ed421a7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/MutatingParallelIteratorFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/MutatingParallelIteratorFactory.java
@@ -32,9 +32,6 @@ import org.apache.phoenix.iterate.ParallelIteratorFactory;
import org.apache.phoenix.iterate.PeekingResultIterator;
import org.apache.phoenix.iterate.ResultIterator;
import org.apache.phoenix.jdbc.PhoenixConnection;
-import org.apache.phoenix.query.ConnectionQueryServices;
-import org.apache.phoenix.query.QueryServices;
-import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.tuple.SingleKeyValueTuple;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PLong;
@@ -64,20 +61,7 @@ public abstract class MutatingParallelIteratorFactory implements ParallelIterato
long totalRowCount = state.getUpdateCount();
if (clonedConnection.getAutoCommit()) {
clonedConnection.getMutationState().join(state);
- clonedConnection.commit();
- ConnectionQueryServices services = clonedConnection.getQueryServices();
- int maxSize = services.getProps().getInt(QueryServices.MAX_MUTATION_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE);
- /*
- * Everything that was mutated as part of the clonedConnection has been committed. However, we want to
- * report the mutation work done using this clonedConnection as part of the overall mutation work of the
- * parent connection. So we need to set those metrics in the empty mutation state so that they could be
- * combined with the parent connection's mutation metrics (as part of combining mutation state) in the
- * close() method of the iterator being returned. Don't combine the read metrics in parent context yet
- * though because they are possibly being concurrently modified by other threads at this stage. Instead we
- * will get hold of the read metrics when all the mutating iterators are done.
- */
- state = MutationState.emptyMutationState(maxSize, clonedConnection);
- state.getMutationMetricQueue().combineMetricQueues(clonedConnection.getMutationState().getMutationMetricQueue());
+ state = clonedConnection.getMutationState();
}
final MutationState finalState = state;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/423bd134/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 2806acd..715e7c4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -700,6 +700,9 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
@Override
public void close() throws SQLException {
+ if (allFutures.isEmpty()) {
+ return;
+ }
// Don't call cancel on already started work, as it causes the HConnection
// to get into a funk. Instead, just cancel queued work.
boolean cancelledWork = false;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/423bd134/phoenix-core/src/main/java/org/apache/phoenix/iterate/MergeSortResultIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/MergeSortResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/MergeSortResultIterator.java
index e076a40..961e766 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/MergeSortResultIterator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/MergeSortResultIterator.java
@@ -24,8 +24,6 @@ import java.util.PriorityQueue;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.schema.tuple.Tuple;
-import org.apache.phoenix.util.SQLCloseables;
-import org.apache.phoenix.util.ServerUtil;
/**
* Base class for a ResultIterator that does a merge sort on the list of iterators provided.
@@ -43,28 +41,7 @@ public abstract class MergeSortResultIterator implements PeekingResultIterator {
@Override
public void close() throws SQLException {
- SQLException toThrow = null;
- try {
- if (resultIterators != null) {
- resultIterators.close();
- }
- } catch (Exception e) {
- toThrow = ServerUtil.parseServerException(e);
- } finally {
- try {
- if (minHeap != null) {
- SQLCloseables.closeAllQuietly(minHeap);
- }
- } catch (Exception e) {
- if (toThrow == null) {
- toThrow = ServerUtil.parseServerException(e);
- } else {
- toThrow.setNextException(ServerUtil.parseServerException(e));
- }
- } finally {
- if (toThrow != null) { throw toThrow; }
- }
- }
+ resultIterators.close();
}
abstract protected int compare(Tuple t1, Tuple t2);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/423bd134/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 29b9756..daa7809 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -2487,7 +2487,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
@Override
public MutationState updateData(MutationPlan plan) throws SQLException {
- return plan.execute();
+ MutationState state = plan.execute();
+ plan.getContext().getConnection().commit();
+ return state;
}
@Override
[46/50] [abbrv] phoenix git commit: PHOENIX-2086 CONSTRAINT_VIOLATION
error code should use sqlstate 23, not 22
Posted by sa...@apache.org.
PHOENIX-2086 CONSTRAINT_VIOLATION error code should use sqlstate 23, not 22
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ffe24906
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ffe24906
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ffe24906
Branch: refs/heads/4.x-HBase-1.0
Commit: ffe2490623ef16dcdac1e2ee54c1c7dff207d885
Parents: ef34d1d
Author: Samarth <sa...@salesforce.com>
Authored: Tue Jan 5 12:59:56 2016 -0800
Committer: Samarth <sa...@salesforce.com>
Committed: Tue Jan 5 12:59:56 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/phoenix/exception/SQLExceptionCode.java | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ffe24906/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index 228f06f..38e8ea0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -83,11 +83,8 @@ public enum SQLExceptionCode {
SINGLE_ROW_SUBQUERY_RETURNS_MULTIPLE_ROWS(215, "22015", "Single-row sub-query returns more than one row."),
SUBQUERY_RETURNS_DIFFERENT_NUMBER_OF_FIELDS(216, "22016", "Sub-query must return the same number of fields as the left-hand-side expression of 'IN'."),
AMBIGUOUS_JOIN_CONDITION(217, "22017", "Ambiguous or non-equi join condition specified. Consider using table list with where clause."),
- CONSTRAINT_VIOLATION(218, "22018", "Constraint violation."),
+ CONSTRAINT_VIOLATION(218, "23018", "Constraint violation."),
- /**
- * Constraint Violation (errorcode 03, sqlstate 23)
- */
CONCURRENT_TABLE_MUTATION(301, "23000", "Concurrent modification to table.", new Factory() {
@Override
public SQLException newException(SQLExceptionInfo info) {
[38/50] [abbrv] phoenix git commit: PHOENIX-2557 Track unfree memory
for server-side cache
Posted by sa...@apache.org.
PHOENIX-2557 Track unfree memory for server-side cache
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/31a414c8
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/31a414c8
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/31a414c8
Branch: refs/heads/4.x-HBase-1.0
Commit: 31a414c84e64a1de366703cb1faa25c9e506d1a3
Parents: 139e0d7
Author: James Taylor <jt...@salesforce.com>
Authored: Sun Jan 3 15:51:40 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Jan 4 09:36:30 2016 -0800
----------------------------------------------------------------------
.../ConnectionQueryServicesTestImpl.java | 4 +-
.../org/apache/phoenix/cache/GlobalCache.java | 23 ++-
.../org/apache/phoenix/cache/TenantCache.java | 4 +-
.../apache/phoenix/cache/TenantCacheImpl.java | 28 ++-
.../coprocessor/MetaDataEndpointImpl.java | 5 +-
.../coprocessor/ServerCachingEndpointImpl.java | 7 +-
.../coprocessor/generated/MetaDataProtos.java | 171 +++++++++++++++----
.../phoenix/memory/GlobalMemoryManager.java | 6 +
.../phoenix/query/ConnectionQueryServices.java | 2 +-
.../query/ConnectionQueryServicesImpl.java | 39 +++--
.../query/ConnectionlessQueryServicesImpl.java | 9 +-
.../query/DelegateConnectionQueryServices.java | 4 +-
.../apache/phoenix/cache/TenantCacheTest.java | 100 +++++++++++
phoenix-protocol/src/main/MetaDataService.proto | 1 +
14 files changed, 332 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a414c8/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
index d941fd5..f5d7f18 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
@@ -81,7 +81,9 @@ public class ConnectionQueryServicesTestImpl extends ConnectionQueryServicesImpl
Set<PhoenixConnection> connections = this.connections;
this.connections = Sets.newHashSet();
SQLCloseables.closeAll(connections);
- clearCache();
+ long unfreedBytes = clearCache();
+ // FIXME: once PHOENIX-2556 is fixed, comment this back in
+ // assertEquals(0,unfreedBytes);
} finally {
super.close();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a414c8/phoenix-core/src/main/java/org/apache/phoenix/cache/GlobalCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/cache/GlobalCache.java b/phoenix-core/src/main/java/org/apache/phoenix/cache/GlobalCache.java
index af5438c..7d04f5b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/GlobalCache.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/GlobalCache.java
@@ -22,6 +22,7 @@ import static org.apache.phoenix.query.QueryServices.MAX_MEMORY_SIZE_ATTRIB;
import static org.apache.phoenix.query.QueryServices.MAX_MEMORY_WAIT_MS_ATTRIB;
import static org.apache.phoenix.query.QueryServices.MAX_TENANT_MEMORY_PERC_ATTRIB;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
@@ -29,6 +30,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.memory.ChildMemoryManager;
import org.apache.phoenix.memory.GlobalMemoryManager;
@@ -36,6 +38,8 @@ import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PMetaDataEntity;
import org.apache.phoenix.util.SizedUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
@@ -51,6 +55,7 @@ import com.google.common.cache.Weigher;
* @since 0.1
*/
public class GlobalCache extends TenantCacheImpl {
+ private static final Logger logger = LoggerFactory.getLogger(GlobalCache.class);
private static volatile GlobalCache INSTANCE;
private final Configuration config;
@@ -59,8 +64,24 @@ public class GlobalCache extends TenantCacheImpl {
// Cache for lastest PTable for a given Phoenix table
private volatile Cache<ImmutableBytesPtr,PMetaDataEntity> metaDataCache;
- public void clearTenantCache() {
+ public long clearTenantCache() {
+ long unfreedBytes = getMemoryManager().getMaxMemory() - getMemoryManager().getAvailableMemory();
+ if (unfreedBytes != 0 && logger.isDebugEnabled()) {
+ logger.debug("Found " + (getMemoryManager().getMaxMemory() - getMemoryManager().getAvailableMemory()) + " bytes not freed from global cache");
+ }
+ removeAllServerCache();
+ for (Map.Entry<ImmutableBytesWritable, TenantCache> entry : perTenantCacheMap.entrySet()) {
+ TenantCache cache = entry.getValue();
+ long unfreedTenantBytes = cache.getMemoryManager().getMaxMemory() - cache.getMemoryManager().getAvailableMemory();
+ if (unfreedTenantBytes != 0 && logger.isDebugEnabled()) {
+ ImmutableBytesWritable cacheId = entry.getKey();
+ logger.debug("Found " + unfreedTenantBytes + " bytes not freed for tenant " + Bytes.toStringBinary(cacheId.get(), cacheId.getOffset(), cacheId.getLength()));
+ }
+ unfreedBytes += unfreedTenantBytes;
+ cache.removeAllServerCache();
+ }
perTenantCacheMap.clear();
+ return unfreedBytes;
}
public Cache<ImmutableBytesPtr,PMetaDataEntity> getMetaDataCache() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a414c8/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCache.java b/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCache.java
index c7cd58f..5c33967 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCache.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCache.java
@@ -21,7 +21,6 @@ import java.io.Closeable;
import java.sql.SQLException;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
import org.apache.phoenix.coprocessor.ServerCachingProtocol.ServerCacheFactory;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.memory.MemoryManager;
@@ -38,5 +37,6 @@ public interface TenantCache {
MemoryManager getMemoryManager();
Closeable getServerCache(ImmutableBytesPtr cacheId);
Closeable addServerCache(ImmutableBytesPtr cacheId, ImmutableBytesWritable cachePtr, byte[] txState, ServerCacheFactory cacheFactory) throws SQLException;
- void removeServerCache(ImmutableBytesPtr cacheId) throws SQLException;
+ void removeServerCache(ImmutableBytesPtr cacheId);
+ void removeAllServerCache();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a414c8/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
index 6ef7a6f..658b4cc 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
@@ -29,6 +29,7 @@ import org.apache.phoenix.memory.MemoryManager;
import org.apache.phoenix.memory.MemoryManager.MemoryChunk;
import org.apache.phoenix.util.Closeables;
+import com.google.common.base.Ticker;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
@@ -45,11 +46,30 @@ import com.google.common.cache.RemovalNotification;
public class TenantCacheImpl implements TenantCache {
private final int maxTimeToLiveMs;
private final MemoryManager memoryManager;
+ private final Ticker ticker;
private volatile Cache<ImmutableBytesPtr, Closeable> serverCaches;
public TenantCacheImpl(MemoryManager memoryManager, int maxTimeToLiveMs) {
+ this(memoryManager, maxTimeToLiveMs, Ticker.systemTicker());
+ }
+
+ public TenantCacheImpl(MemoryManager memoryManager, int maxTimeToLiveMs, Ticker ticker) {
this.memoryManager = memoryManager;
this.maxTimeToLiveMs = maxTimeToLiveMs;
+ this.ticker = ticker;
+ }
+
+ public Ticker getTicker() {
+ return ticker;
+ }
+
+ // For testing
+ public void cleanUp() {
+ synchronized(this) {
+ if (serverCaches != null) {
+ serverCaches.cleanUp();
+ }
+ }
}
@Override
@@ -64,6 +84,7 @@ public class TenantCacheImpl implements TenantCache {
if (serverCaches == null) {
serverCaches = CacheBuilder.newBuilder()
.expireAfterAccess(maxTimeToLiveMs, TimeUnit.MILLISECONDS)
+ .ticker(getTicker())
.removalListener(new RemovalListener<ImmutableBytesPtr, Closeable>(){
@Override
public void onRemoval(RemovalNotification<ImmutableBytesPtr, Closeable> notification) {
@@ -99,7 +120,12 @@ public class TenantCacheImpl implements TenantCache {
}
@Override
- public void removeServerCache(ImmutableBytesPtr cacheId) throws SQLException {
+ public void removeServerCache(ImmutableBytesPtr cacheId) {
getServerCaches().invalidate(cacheId);
}
+
+ @Override
+ public void removeAllServerCache() {
+ getServerCaches().invalidateAll();
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a414c8/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index ea4a7e1..13876a0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -2667,7 +2667,10 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
Cache<ImmutableBytesPtr, PMetaDataEntity> metaDataCache =
GlobalCache.getInstance(this.env).getMetaDataCache();
metaDataCache.invalidateAll();
- cache.clearTenantCache();
+ long unfreedBytes = cache.clearTenantCache();
+ ClearCacheResponse.Builder builder = ClearCacheResponse.newBuilder();
+ builder.setUnfreedBytes(unfreedBytes);
+ done.run(builder.build());
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a414c8/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
index 9f3bdb4..bf889d5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingEndpointImpl.java
@@ -18,7 +18,6 @@
package org.apache.phoenix.coprocessor;
import java.io.IOException;
-import java.sql.SQLException;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
@@ -91,11 +90,7 @@ public class ServerCachingEndpointImpl extends ServerCachingService implements C
tenantId = new ImmutableBytesPtr(request.getTenantId().toByteArray());
}
TenantCache tenantCache = GlobalCache.getTenantCache(this.env, tenantId);
- try {
- tenantCache.removeServerCache(new ImmutableBytesPtr(request.getCacheId().toByteArray()));
- } catch (SQLException e) {
- ProtobufUtil.setControllerException(controller, new IOException(e));
- }
+ tenantCache.removeServerCache(new ImmutableBytesPtr(request.getCacheId().toByteArray()));
RemoveServerCacheResponse.Builder responseBuilder = RemoveServerCacheResponse.newBuilder();
responseBuilder.setReturn(true);
RemoveServerCacheResponse result = responseBuilder.build();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a414c8/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
index a121d28..dc5726a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
@@ -8767,6 +8767,16 @@ public final class MetaDataProtos {
public interface ClearCacheResponseOrBuilder
extends com.google.protobuf.MessageOrBuilder {
+
+ // optional int64 unfreedBytes = 1;
+ /**
+ * <code>optional int64 unfreedBytes = 1;</code>
+ */
+ boolean hasUnfreedBytes();
+ /**
+ * <code>optional int64 unfreedBytes = 1;</code>
+ */
+ long getUnfreedBytes();
}
/**
* Protobuf type {@code ClearCacheResponse}
@@ -8801,6 +8811,7 @@ public final class MetaDataProtos {
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
throws com.google.protobuf.InvalidProtocolBufferException {
initFields();
+ int mutable_bitField0_ = 0;
com.google.protobuf.UnknownFieldSet.Builder unknownFields =
com.google.protobuf.UnknownFieldSet.newBuilder();
try {
@@ -8818,6 +8829,11 @@ public final class MetaDataProtos {
}
break;
}
+ case 8: {
+ bitField0_ |= 0x00000001;
+ unfreedBytes_ = input.readInt64();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -8857,7 +8873,25 @@ public final class MetaDataProtos {
return PARSER;
}
+ private int bitField0_;
+ // optional int64 unfreedBytes = 1;
+ public static final int UNFREEDBYTES_FIELD_NUMBER = 1;
+ private long unfreedBytes_;
+ /**
+ * <code>optional int64 unfreedBytes = 1;</code>
+ */
+ public boolean hasUnfreedBytes() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional int64 unfreedBytes = 1;</code>
+ */
+ public long getUnfreedBytes() {
+ return unfreedBytes_;
+ }
+
private void initFields() {
+ unfreedBytes_ = 0L;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -8871,6 +8905,9 @@ public final class MetaDataProtos {
public void writeTo(com.google.protobuf.CodedOutputStream output)
throws java.io.IOException {
getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeInt64(1, unfreedBytes_);
+ }
getUnknownFields().writeTo(output);
}
@@ -8880,6 +8917,10 @@ public final class MetaDataProtos {
if (size != -1) return size;
size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt64Size(1, unfreedBytes_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -8903,6 +8944,11 @@ public final class MetaDataProtos {
org.apache.phoenix.coprocessor.generated.MetaDataProtos.ClearCacheResponse other = (org.apache.phoenix.coprocessor.generated.MetaDataProtos.ClearCacheResponse) obj;
boolean result = true;
+ result = result && (hasUnfreedBytes() == other.hasUnfreedBytes());
+ if (hasUnfreedBytes()) {
+ result = result && (getUnfreedBytes()
+ == other.getUnfreedBytes());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -8916,6 +8962,10 @@ public final class MetaDataProtos {
}
int hash = 41;
hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasUnfreedBytes()) {
+ hash = (37 * hash) + UNFREEDBYTES_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getUnfreedBytes());
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -9025,6 +9075,8 @@ public final class MetaDataProtos {
public Builder clear() {
super.clear();
+ unfreedBytes_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000001);
return this;
}
@@ -9051,6 +9103,13 @@ public final class MetaDataProtos {
public org.apache.phoenix.coprocessor.generated.MetaDataProtos.ClearCacheResponse buildPartial() {
org.apache.phoenix.coprocessor.generated.MetaDataProtos.ClearCacheResponse result = new org.apache.phoenix.coprocessor.generated.MetaDataProtos.ClearCacheResponse(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.unfreedBytes_ = unfreedBytes_;
+ result.bitField0_ = to_bitField0_;
onBuilt();
return result;
}
@@ -9066,6 +9125,9 @@ public final class MetaDataProtos {
public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.MetaDataProtos.ClearCacheResponse other) {
if (other == org.apache.phoenix.coprocessor.generated.MetaDataProtos.ClearCacheResponse.getDefaultInstance()) return this;
+ if (other.hasUnfreedBytes()) {
+ setUnfreedBytes(other.getUnfreedBytes());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -9091,6 +9153,40 @@ public final class MetaDataProtos {
}
return this;
}
+ private int bitField0_;
+
+ // optional int64 unfreedBytes = 1;
+ private long unfreedBytes_ ;
+ /**
+ * <code>optional int64 unfreedBytes = 1;</code>
+ */
+ public boolean hasUnfreedBytes() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional int64 unfreedBytes = 1;</code>
+ */
+ public long getUnfreedBytes() {
+ return unfreedBytes_;
+ }
+ /**
+ * <code>optional int64 unfreedBytes = 1;</code>
+ */
+ public Builder setUnfreedBytes(long value) {
+ bitField0_ |= 0x00000001;
+ unfreedBytes_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional int64 unfreedBytes = 1;</code>
+ */
+ public Builder clearUnfreedBytes() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ unfreedBytes_ = 0L;
+ onChanged();
+ return this;
+ }
// @@protoc_insertion_point(builder_scope:ClearCacheResponse)
}
@@ -12285,42 +12381,43 @@ public final class MetaDataProtos {
"ateIndexStateRequest\022\036\n\026tableMetadataMut" +
"ations\030\001 \003(\014\022\025\n\rclientVersion\030\002 \001(\005\"*\n\021C",
"learCacheRequest\022\025\n\rclientVersion\030\001 \001(\005\"" +
- "\024\n\022ClearCacheResponse\"*\n\021GetVersionReque" +
- "st\022\025\n\rclientVersion\030\001 \001(\005\"%\n\022GetVersionR" +
- "esponse\022\017\n\007version\030\001 \002(\003\"\205\001\n\032ClearTableF" +
- "romCacheRequest\022\020\n\010tenantId\030\001 \002(\014\022\022\n\nsch" +
- "emaName\030\002 \002(\014\022\021\n\ttableName\030\003 \002(\014\022\027\n\017clie" +
- "ntTimestamp\030\004 \002(\003\022\025\n\rclientVersion\030\005 \001(\005" +
- "\"\035\n\033ClearTableFromCacheResponse*\365\002\n\014Muta" +
- "tionCode\022\030\n\024TABLE_ALREADY_EXISTS\020\000\022\023\n\017TA" +
- "BLE_NOT_FOUND\020\001\022\024\n\020COLUMN_NOT_FOUND\020\002\022\031\n",
- "\025COLUMN_ALREADY_EXISTS\020\003\022\035\n\031CONCURRENT_T" +
- "ABLE_MUTATION\020\004\022\027\n\023TABLE_NOT_IN_REGION\020\005" +
- "\022\025\n\021NEWER_TABLE_FOUND\020\006\022\034\n\030UNALLOWED_TAB" +
- "LE_MUTATION\020\007\022\021\n\rNO_PK_COLUMNS\020\010\022\032\n\026PARE" +
- "NT_TABLE_NOT_FOUND\020\t\022\033\n\027FUNCTION_ALREADY" +
- "_EXISTS\020\n\022\026\n\022FUNCTION_NOT_FOUND\020\013\022\030\n\024NEW" +
- "ER_FUNCTION_FOUND\020\014\022\032\n\026FUNCTION_NOT_IN_R" +
- "EGION\020\r2\304\005\n\017MetaDataService\022/\n\010getTable\022" +
- "\020.GetTableRequest\032\021.MetaDataResponse\0227\n\014" +
- "getFunctions\022\024.GetFunctionsRequest\032\021.Met",
- "aDataResponse\0225\n\013createTable\022\023.CreateTab" +
- "leRequest\032\021.MetaDataResponse\022;\n\016createFu" +
- "nction\022\026.CreateFunctionRequest\032\021.MetaDat" +
- "aResponse\0221\n\tdropTable\022\021.DropTableReques" +
- "t\032\021.MetaDataResponse\0227\n\014dropFunction\022\024.D" +
- "ropFunctionRequest\032\021.MetaDataResponse\0221\n" +
- "\taddColumn\022\021.AddColumnRequest\032\021.MetaData" +
- "Response\0223\n\ndropColumn\022\022.DropColumnReque" +
- "st\032\021.MetaDataResponse\022?\n\020updateIndexStat" +
- "e\022\030.UpdateIndexStateRequest\032\021.MetaDataRe",
- "sponse\0225\n\nclearCache\022\022.ClearCacheRequest" +
- "\032\023.ClearCacheResponse\0225\n\ngetVersion\022\022.Ge" +
- "tVersionRequest\032\023.GetVersionResponse\022P\n\023" +
- "clearTableFromCache\022\033.ClearTableFromCach" +
- "eRequest\032\034.ClearTableFromCacheResponseBB" +
- "\n(org.apache.phoenix.coprocessor.generat" +
- "edB\016MetaDataProtosH\001\210\001\001\240\001\001"
+ "*\n\022ClearCacheResponse\022\024\n\014unfreedBytes\030\001 " +
+ "\001(\003\"*\n\021GetVersionRequest\022\025\n\rclientVersio" +
+ "n\030\001 \001(\005\"%\n\022GetVersionResponse\022\017\n\007version" +
+ "\030\001 \002(\003\"\205\001\n\032ClearTableFromCacheRequest\022\020\n" +
+ "\010tenantId\030\001 \002(\014\022\022\n\nschemaName\030\002 \002(\014\022\021\n\tt" +
+ "ableName\030\003 \002(\014\022\027\n\017clientTimestamp\030\004 \002(\003\022" +
+ "\025\n\rclientVersion\030\005 \001(\005\"\035\n\033ClearTableFrom" +
+ "CacheResponse*\365\002\n\014MutationCode\022\030\n\024TABLE_" +
+ "ALREADY_EXISTS\020\000\022\023\n\017TABLE_NOT_FOUND\020\001\022\024\n",
+ "\020COLUMN_NOT_FOUND\020\002\022\031\n\025COLUMN_ALREADY_EX" +
+ "ISTS\020\003\022\035\n\031CONCURRENT_TABLE_MUTATION\020\004\022\027\n" +
+ "\023TABLE_NOT_IN_REGION\020\005\022\025\n\021NEWER_TABLE_FO" +
+ "UND\020\006\022\034\n\030UNALLOWED_TABLE_MUTATION\020\007\022\021\n\rN" +
+ "O_PK_COLUMNS\020\010\022\032\n\026PARENT_TABLE_NOT_FOUND" +
+ "\020\t\022\033\n\027FUNCTION_ALREADY_EXISTS\020\n\022\026\n\022FUNCT" +
+ "ION_NOT_FOUND\020\013\022\030\n\024NEWER_FUNCTION_FOUND\020" +
+ "\014\022\032\n\026FUNCTION_NOT_IN_REGION\020\r2\304\005\n\017MetaDa" +
+ "taService\022/\n\010getTable\022\020.GetTableRequest\032" +
+ "\021.MetaDataResponse\0227\n\014getFunctions\022\024.Get",
+ "FunctionsRequest\032\021.MetaDataResponse\0225\n\013c" +
+ "reateTable\022\023.CreateTableRequest\032\021.MetaDa" +
+ "taResponse\022;\n\016createFunction\022\026.CreateFun" +
+ "ctionRequest\032\021.MetaDataResponse\0221\n\tdropT" +
+ "able\022\021.DropTableRequest\032\021.MetaDataRespon" +
+ "se\0227\n\014dropFunction\022\024.DropFunctionRequest" +
+ "\032\021.MetaDataResponse\0221\n\taddColumn\022\021.AddCo" +
+ "lumnRequest\032\021.MetaDataResponse\0223\n\ndropCo" +
+ "lumn\022\022.DropColumnRequest\032\021.MetaDataRespo" +
+ "nse\022?\n\020updateIndexState\022\030.UpdateIndexSta",
+ "teRequest\032\021.MetaDataResponse\0225\n\nclearCac" +
+ "he\022\022.ClearCacheRequest\032\023.ClearCacheRespo" +
+ "nse\0225\n\ngetVersion\022\022.GetVersionRequest\032\023." +
+ "GetVersionResponse\022P\n\023clearTableFromCach" +
+ "e\022\033.ClearTableFromCacheRequest\032\034.ClearTa" +
+ "bleFromCacheResponseBB\n(org.apache.phoen" +
+ "ix.coprocessor.generatedB\016MetaDataProtos" +
+ "H\001\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -12398,7 +12495,7 @@ public final class MetaDataProtos {
internal_static_ClearCacheResponse_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_ClearCacheResponse_descriptor,
- new java.lang.String[] { });
+ new java.lang.String[] { "UnfreedBytes", });
internal_static_GetVersionRequest_descriptor =
getDescriptor().getMessageTypes().get(12);
internal_static_GetVersionRequest_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a414c8/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java b/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
index e70b35f..74f2c5f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
@@ -73,6 +73,7 @@ public class GlobalMemoryManager implements MemoryManager {
synchronized(sync) {
while (usedMemoryBytes + minBytes > maxMemoryBytes) { // Only wait if minBytes not available
try {
+ logger.debug("Waiting for " + (usedMemoryBytes + minBytes - maxMemoryBytes) + " bytes to be free");
long remainingWaitTimeMs = maxWaitMs - (System.currentTimeMillis() - startTimeMs);
if (remainingWaitTimeMs <= 0) { // Ran out of time waiting for some memory to get freed up
throw new InsufficientMemoryException("Requested memory of " + minBytes + " bytes could not be allocated. Using memory of " + usedMemoryBytes + " bytes from global pool of " + maxMemoryBytes + " bytes after waiting for " + maxWaitMs + "ms.");
@@ -109,12 +110,15 @@ public class GlobalMemoryManager implements MemoryManager {
private class GlobalMemoryChunk implements MemoryChunk {
private volatile long size;
+ //private volatile String stack;
private GlobalMemoryChunk(long size) {
if (size < 0) {
throw new IllegalStateException("Size of memory chunk must be greater than zero, but instead is " + size);
}
this.size = size;
+ // Useful for debugging where a piece of memory was allocated
+ // this.stack = ExceptionUtils.getStackTrace(new Throwable());
}
@Override
@@ -138,6 +142,7 @@ public class GlobalMemoryManager implements MemoryManager {
} else {
allocateBytes(nAdditionalBytes, nAdditionalBytes);
size = nBytes;
+ //this.stack = ExceptionUtils.getStackTrace(new Throwable());
}
}
}
@@ -150,6 +155,7 @@ public class GlobalMemoryManager implements MemoryManager {
try {
if (size > 0) {
logger.warn("Orphaned chunk of " + size + " bytes found during finalize");
+ //logger.warn("Orphaned chunk of " + size + " bytes found during finalize allocated here:\n" + stack);
}
freeMemory();
} finally {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a414c8/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index fc41706..d839fa3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -116,7 +116,7 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
public PTableStats getTableStats(byte[] physicalName, long clientTimeStamp) throws SQLException;
- public void clearCache() throws SQLException;
+ public long clearCache() throws SQLException;
public int getSequenceSaltBuckets();
TransactionSystemClient getTransactionSystemClient();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a414c8/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 1ea7985..29b9756 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -2505,27 +2505,35 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
* @throws SQLException
*/
@Override
- public void clearCache() throws SQLException {
+ public long clearCache() throws SQLException {
try {
SQLException sqlE = null;
HTableInterface htable = this.getTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES);
try {
- htable.coprocessorService(MetaDataService.class, HConstants.EMPTY_START_ROW,
- HConstants.EMPTY_END_ROW, new Batch.Call<MetaDataService, ClearCacheResponse>() {
- @Override
- public ClearCacheResponse call(MetaDataService instance) throws IOException {
- ServerRpcController controller = new ServerRpcController();
- BlockingRpcCallback<ClearCacheResponse> rpcCallback =
- new BlockingRpcCallback<ClearCacheResponse>();
- ClearCacheRequest.Builder builder = ClearCacheRequest.newBuilder();
- builder.setClientVersion(VersionUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER));
- instance.clearCache(controller, builder.build(), rpcCallback);
- if(controller.getFailedOn() != null) {
- throw controller.getFailedOn();
+ final Map<byte[], Long> results =
+ htable.coprocessorService(MetaDataService.class, HConstants.EMPTY_START_ROW,
+ HConstants.EMPTY_END_ROW, new Batch.Call<MetaDataService, Long>() {
+ @Override
+ public Long call(MetaDataService instance) throws IOException {
+ ServerRpcController controller = new ServerRpcController();
+ BlockingRpcCallback<ClearCacheResponse> rpcCallback =
+ new BlockingRpcCallback<ClearCacheResponse>();
+ ClearCacheRequest.Builder builder = ClearCacheRequest.newBuilder();
+ builder.setClientVersion(VersionUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER));
+ instance.clearCache(controller, builder.build(), rpcCallback);
+ if(controller.getFailedOn() != null) {
+ throw controller.getFailedOn();
+ }
+ return rpcCallback.get().getUnfreedBytes();
}
- return rpcCallback.get();
+ });
+ long unfreedBytes = 0;
+ for (Map.Entry<byte[],Long> result : results.entrySet()) {
+ if (result.getValue() != null) {
+ unfreedBytes += result.getValue();
}
- });
+ }
+ return unfreedBytes;
} catch (IOException e) {
throw ServerUtil.parseServerException(e);
} catch (Throwable e) {
@@ -2549,6 +2557,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
} catch (Exception e) {
throw new SQLException(ServerUtil.parseServerException(e));
}
+ return 0;
}
private void flushTable(byte[] tableName) throws SQLException {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a414c8/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index e983a4c..f1ab319 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -83,13 +83,13 @@ import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.SequenceUtil;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
import co.cask.tephra.TransactionManager;
import co.cask.tephra.TransactionSystemClient;
import co.cask.tephra.inmemory.InMemoryTxSystemClient;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
/**
*
@@ -512,7 +512,8 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
}
@Override
- public void clearCache() throws SQLException {
+ public long clearCache() throws SQLException {
+ return 0;
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a414c8/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index ca662be..b56ff85 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -251,8 +251,8 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
@Override
- public void clearCache() throws SQLException {
- getDelegate().clearCache();
+ public long clearCache() throws SQLException {
+ return getDelegate().clearCache();
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a414c8/phoenix-core/src/test/java/org/apache/phoenix/cache/TenantCacheTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/cache/TenantCacheTest.java b/phoenix-core/src/test/java/org/apache/phoenix/cache/TenantCacheTest.java
new file mode 100644
index 0000000..ac2a850
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/cache/TenantCacheTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.cache;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.sql.SQLException;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.coprocessor.ServerCachingProtocol.ServerCacheFactory;
+import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.memory.GlobalMemoryManager;
+import org.apache.phoenix.memory.MemoryManager.MemoryChunk;
+import org.apache.phoenix.util.ByteUtil;
+import org.junit.Test;
+
+import com.google.common.base.Ticker;
+
+public class TenantCacheTest {
+
+ @Test
+ public void testInvalidateClosesMemoryChunk() throws SQLException {
+ int maxServerCacheTimeToLive = 10000;
+ long maxBytes = 1000;
+ int maxWaitMs = 1000;
+ GlobalMemoryManager memoryManager = new GlobalMemoryManager(maxBytes, maxWaitMs);
+ TenantCacheImpl newTenantCache = new TenantCacheImpl(memoryManager, maxServerCacheTimeToLive);
+ ImmutableBytesPtr cacheId = new ImmutableBytesPtr(Bytes.toBytes("a"));
+ ImmutableBytesWritable cachePtr = new ImmutableBytesWritable(Bytes.toBytes("a"));
+ newTenantCache.addServerCache(cacheId, cachePtr, ByteUtil.EMPTY_BYTE_ARRAY, cacheFactory);
+ assertEquals(maxBytes-1, memoryManager.getAvailableMemory());
+ newTenantCache.removeServerCache(cacheId);
+ assertEquals(maxBytes, memoryManager.getAvailableMemory());
+ }
+
+ @Test
+ public void testTimeoutClosesMemoryChunk() throws Exception {
+ int maxServerCacheTimeToLive = 10;
+ long maxBytes = 1000;
+ int maxWaitMs = 10;
+ GlobalMemoryManager memoryManager = new GlobalMemoryManager(maxBytes, maxWaitMs);
+ ManualTicker ticker = new ManualTicker();
+ TenantCacheImpl cache = new TenantCacheImpl(memoryManager, maxServerCacheTimeToLive, ticker);
+ ImmutableBytesPtr cacheId1 = new ImmutableBytesPtr(Bytes.toBytes("a"));
+ ImmutableBytesWritable cachePtr = new ImmutableBytesWritable(Bytes.toBytes("a"));
+ cache.addServerCache(cacheId1, cachePtr, ByteUtil.EMPTY_BYTE_ARRAY, cacheFactory);
+ assertEquals(maxBytes-1, memoryManager.getAvailableMemory());
+ ticker.time += (maxServerCacheTimeToLive + 1) * 1000000;
+ cache.cleanUp();
+ assertEquals(maxBytes, memoryManager.getAvailableMemory());
+ }
+
+ public static class ManualTicker extends Ticker {
+ public long time = 0;
+
+ @Override
+ public long read() {
+ return time;
+ }
+
+ }
+
+ public static ServerCacheFactory cacheFactory = new ServerCacheFactory() {
+
+ @Override
+ public void readFields(DataInput arg0) throws IOException {
+ }
+
+ @Override
+ public void write(DataOutput arg0) throws IOException {
+ }
+
+ @Override
+ public Closeable newCache(ImmutableBytesWritable cachePtr, byte[] txState, MemoryChunk chunk)
+ throws SQLException {
+ return chunk;
+ }
+
+ };
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a414c8/phoenix-protocol/src/main/MetaDataService.proto
----------------------------------------------------------------------
diff --git a/phoenix-protocol/src/main/MetaDataService.proto b/phoenix-protocol/src/main/MetaDataService.proto
index c265158..c631512 100644
--- a/phoenix-protocol/src/main/MetaDataService.proto
+++ b/phoenix-protocol/src/main/MetaDataService.proto
@@ -119,6 +119,7 @@ message ClearCacheRequest {
}
message ClearCacheResponse {
+ optional int64 unfreedBytes = 1;
}
message GetVersionRequest {
[45/50] [abbrv] phoenix git commit: PHOENIX-2039 ROUND over numeric
in GROUP BY always returns null (testcase only)-Ram
Posted by sa...@apache.org.
PHOENIX-2039 ROUND over numeric in GROUP BY always returns null (testcase
only)-Ram
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ef34d1d4
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ef34d1d4
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ef34d1d4
Branch: refs/heads/4.x-HBase-1.0
Commit: ef34d1d44fa43087e6b57ffd0185ba3d85f97f9a
Parents: 316871b
Author: ramkrishna <ra...@gmail.com>
Authored: Tue Jan 5 17:48:06 2016 +0530
Committer: ramkrishna <ra...@gmail.com>
Committed: Tue Jan 5 22:13:02 2016 +0530
----------------------------------------------------------------------
.../RoundFloorCeilFunctionsEnd2EndIT.java | 38 ++++++++++++++++++++
1 file changed, 38 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ef34d1d4/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFunctionsEnd2EndIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFunctionsEnd2EndIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFunctionsEnd2EndIT.java
index 42635c6..cd00354 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFunctionsEnd2EndIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFunctionsEnd2EndIT.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.end2end;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.apache.phoenix.util.TestUtil.closeStmtAndConn;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -27,6 +28,7 @@ import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
+import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
import java.util.Properties;
@@ -36,6 +38,8 @@ import org.apache.phoenix.expression.function.FloorFunction;
import org.apache.phoenix.expression.function.RoundFunction;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.DateUtil;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.PropertiesUtil;
import org.junit.Before;
import org.junit.Test;
@@ -554,4 +558,38 @@ public class RoundFloorCeilFunctionsEnd2EndIT extends BaseHBaseManagedTimeIT {
}
}
+ @Test
+ public void testRoundOffFunction() throws SQLException {
+ long ts = nextTimestamp();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String ddl = "create table round_test(k bigint primary key)";
+ conn.createStatement().execute(ddl);
+ conn.close();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30));
+ conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("upsert into round_test values(1380603308885)");
+ stmt.execute();
+ conn.commit();
+ conn.close();
+
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 40));
+ conn = DriverManager.getConnection(getUrl(), props);
+ ResultSet rs;
+ stmt = conn.prepareStatement("select round(k/1000000,0) from round_test");
+ rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(1380603, rs.getLong(1));
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 40));
+ conn = DriverManager.getConnection(getUrl(), props);
+ stmt = conn.prepareStatement("select round(k/1000000,0) x from round_test group by x");
+ rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(1380603, rs.getLong(1));
+ }
+
}
[22/50] [abbrv] phoenix git commit: PHOENIX-2528 Pherf JVM doesn't
exit due to un-closed ThreadPool (elserj)
Posted by sa...@apache.org.
PHOENIX-2528 Pherf JVM doesn't exit due to un-closed ThreadPool (elserj)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8b1a7321
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8b1a7321
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8b1a7321
Branch: refs/heads/4.x-HBase-1.0
Commit: 8b1a7321e0de6bec93566de705bda797eee49cfd
Parents: 7ca3698
Author: Mujtaba <mu...@apache.org>
Authored: Tue Dec 22 15:19:15 2015 -0800
Committer: Mujtaba <mu...@apache.org>
Committed: Tue Dec 22 15:19:15 2015 -0800
----------------------------------------------------------------------
.../main/java/org/apache/phoenix/pherf/Pherf.java | 18 ++++++++++--------
.../apache/phoenix/pherf/util/ResourceList.java | 3 ++-
.../phoenix/pherf/workload/WriteWorkload.java | 1 +
3 files changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8b1a7321/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
index b84183b..154d6ff 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
@@ -254,11 +254,17 @@ public class Pherf {
// Schema and Data Load
if (preLoadData) {
logger.info("\nStarting Data Load...");
- WriteWorkload workload = new WriteWorkload(parser, generateStatistics);
- workloadExecutor.add(workload);
+ Workload workload = new WriteWorkload(parser, generateStatistics);
+ try {
+ workloadExecutor.add(workload);
- // Wait for dataLoad to complete
- workloadExecutor.get(workload);
+ // Wait for dataLoad to complete
+ workloadExecutor.get(workload);
+ } finally {
+ if (null != workload) {
+ workload.complete();
+ }
+ }
} else {
logger.info(
"\nSKIPPED: Data Load and schema creation as -l argument not specified");
@@ -290,10 +296,6 @@ public class Pherf {
if (workloadExecutor != null) {
logger.info("Run completed. Shutting down thread pool.");
workloadExecutor.shutdown();
- if (preLoadData) {
- System.exit(0);
- }
-
}
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8b1a7321/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/ResourceList.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/ResourceList.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/ResourceList.java
index 5359c35..037426e 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/ResourceList.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/ResourceList.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.pherf.util;
+import org.apache.commons.lang3.StringUtils;
import org.apache.phoenix.pherf.exception.PherfException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -108,7 +109,7 @@ public class ResourceList {
final String element,
final Pattern pattern) {
final List<String> retVal = new ArrayList<>();
- if (element.equals("")) {
+ if (StringUtils.isBlank(element)) {
return retVal;
}
final File file = new File(element);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8b1a7321/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
index a35e6e8..e536eb9 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
@@ -134,6 +134,7 @@ public class WriteWorkload implements Workload {
}
@Override public void complete() {
+ pool.shutdownNow();
}
public Runnable execute() throws Exception {
[34/50] [abbrv] phoenix git commit: PHOENIX-2552 Fix hanging Jenkins
builds
Posted by sa...@apache.org.
PHOENIX-2552 Fix hanging Jenkins builds
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/4cc7f3c9
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/4cc7f3c9
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/4cc7f3c9
Branch: refs/heads/4.x-HBase-1.0
Commit: 4cc7f3c9296e64c66fe2fbf08c2829660f4ed788
Parents: 4c99fc2
Author: James Taylor <jt...@salesforce.com>
Authored: Thu Dec 31 09:55:38 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Thu Dec 31 11:18:44 2015 -0800
----------------------------------------------------------------------
.../ConnectionQueryServicesTestImpl.java | 20 ++++++++++++++++++++
.../phoenix/iterate/ScanningResultIterator.java | 10 +++++++---
.../apache/phoenix/jdbc/PhoenixConnection.java | 2 +-
3 files changed, 28 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4cc7f3c9/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
index 2028223..d941fd5 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
@@ -19,10 +19,15 @@ package org.apache.phoenix.end2end;
import java.sql.SQLException;
import java.util.Properties;
+import java.util.Set;
+import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
import org.apache.phoenix.query.ConnectionQueryServicesImpl;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.SQLCloseables;
+
+import com.google.common.collect.Sets;
/**
*
@@ -34,12 +39,24 @@ import org.apache.phoenix.query.QueryServices;
*/
public class ConnectionQueryServicesTestImpl extends ConnectionQueryServicesImpl {
protected int NUM_SLAVES_BASE = 1; // number of slaves for the cluster
+ // Track open connections to free them on close as unit tests don't always do this.
+ private Set<PhoenixConnection> connections = Sets.newHashSet();
public ConnectionQueryServicesTestImpl(QueryServices services, ConnectionInfo info, Properties props) throws SQLException {
super(services, info, props);
}
@Override
+ public void addConnection(PhoenixConnection connection) throws SQLException {
+ connections.add(connection);
+ }
+
+ @Override
+ public void removeConnection(PhoenixConnection connection) throws SQLException {
+ connections.remove(connection);
+ }
+
+ @Override
public void init(String url, Properties props) throws SQLException {
try {
super.init(url, props);
@@ -61,6 +78,9 @@ public class ConnectionQueryServicesTestImpl extends ConnectionQueryServicesImpl
@Override
public void close() throws SQLException {
try {
+ Set<PhoenixConnection> connections = this.connections;
+ this.connections = Sets.newHashSet();
+ SQLCloseables.closeAll(connections);
clearCache();
} finally {
super.close();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4cc7f3c9/phoenix-core/src/main/java/org/apache/phoenix/iterate/ScanningResultIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ScanningResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ScanningResultIterator.java
index b722794..604e1a7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ScanningResultIterator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ScanningResultIterator.java
@@ -28,9 +28,9 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.phoenix.monitoring.CombinableMetric;
import org.apache.phoenix.monitoring.CombinableMetric.NoOpRequestMetric;
import org.apache.phoenix.monitoring.GlobalClientMetrics;
-import org.apache.phoenix.monitoring.CombinableMetric;
import org.apache.phoenix.schema.tuple.ResultTuple;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.util.ServerUtil;
@@ -53,10 +53,14 @@ public class ScanningResultIterator implements ResultIterator {
public Tuple next() throws SQLException {
try {
Result result = scanner.next();
+ if (result == null) {
+ close(); // Free up resources early
+ return null;
+ }
calculateScanSize(result);
- // TODO: use ResultTuple.setResult(result)
+ // TODO: use ResultTuple.setResult(result)?
// Need to create a new one if holding on to it (i.e. OrderedResultIterator)
- return result == null ? null : new ResultTuple(result);
+ return new ResultTuple(result);
} catch (IOException e) {
throw ServerUtil.parseServerException(e);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4cc7f3c9/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
index 2a7b933..d547fb4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
@@ -123,7 +123,7 @@ import co.cask.tephra.TransactionContext;
*
* @since 0.1
*/
-public class PhoenixConnection implements Connection, MetaDataMutated {
+public class PhoenixConnection implements Connection, MetaDataMutated, SQLCloseable {
private final String url;
private final ConnectionQueryServices services;
private final Properties info;
[06/50] [abbrv] phoenix git commit: PHOENIX-2455 Partial results for
a query when PHOENIX-2194 is applied
Posted by sa...@apache.org.
PHOENIX-2455 Partial results for a query when PHOENIX-2194 is applied
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ee813cee
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ee813cee
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ee813cee
Branch: refs/heads/4.x-HBase-1.0
Commit: ee813cee428c6ec6027076a838865849ef44b77d
Parents: 72f80bd
Author: James Taylor <jt...@salesforce.com>
Authored: Sun Dec 13 14:32:29 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Dec 14 11:52:19 2015 -0800
----------------------------------------------------------------------
.../apache/phoenix/end2end/SkipScanQueryIT.java | 31 +++++++++
.../apache/phoenix/filter/SkipScanFilter.java | 3 +-
.../java/org/apache/phoenix/query/KeyRange.java | 48 +++++++++-----
.../java/org/apache/phoenix/util/ScanUtil.java | 2 +-
.../phoenix/filter/SkipScanFilterTest.java | 66 +++++++++++++-------
5 files changed, 108 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ee813cee/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java
index 1937f65..2ade0a4 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java
@@ -110,6 +110,37 @@ public class SkipScanQueryIT extends BaseHBaseManagedTimeIT {
}
@Test
+ public void testSkipScanFilterQuery() throws Exception {
+ String createTableDDL = "CREATE TABLE test" + "(col1 VARCHAR," + "col2 VARCHAR," + "col3 VARCHAR,"
+ + "col4 VARCHAR," + "CONSTRAINT pk " + "PRIMARY KEY (col1,col2,col3,col4))";
+ String upsertQuery = "upsert into test values(?,?,?,?)";
+ String query = "SELECT col1, col2, col3, col4 FROM test WHERE col1 IN ('a','e','f') AND col2 = 'b' AND col4 = '1' ";
+ String[] col1Values = { "a", "e.f", "f" };
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ createTestTable(getUrl(), createTableDDL);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(true);
+ try {
+ PreparedStatement statement = conn.prepareStatement(upsertQuery);
+ for (String col1Value : col1Values) {
+ statement.setString(1, col1Value);
+ statement.setString(2, "b");
+ statement.setString(3, "");
+ statement.setString(4, "1");
+ statement.execute();
+ }
+ ResultSet rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals(rs.getString(1), "a");
+ assertTrue(rs.next());
+ assertEquals(rs.getString(1), "f");
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
public void testSelectAfterUpsertInQuery() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
initSelectAfterUpsertTable(conn);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ee813cee/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
index d1c8532..667b3d7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
@@ -580,8 +580,7 @@ public class SkipScanFilter extends FilterBase implements Writable {
List<KeyRange> orClause = Lists.newArrayListWithExpectedSize(orLen);
slots.add(orClause);
for (int j=0; j<orLen; j++) {
- KeyRange range = new KeyRange();
- range.readFields(in);
+ KeyRange range = KeyRange.read(in);
orClause.add(range);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ee813cee/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java b/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java
index 6618aa5..f4bf793 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java
@@ -119,15 +119,30 @@ public class KeyRange implements Writable {
return getKeyRange(lowerRange, true, upperRange, false);
}
- // TODO: make non public and move to org.apache.phoenix.type soon
- public static KeyRange getKeyRange(byte[] lowerRange, boolean lowerInclusive,
+ private static KeyRange getSingleton(byte[] lowerRange, boolean lowerInclusive,
byte[] upperRange, boolean upperInclusive) {
if (lowerRange == null || upperRange == null) {
return EMPTY_RANGE;
}
- // Need to treat null differently for a point range
- if (lowerRange.length == 0 && upperRange.length == 0 && lowerInclusive && upperInclusive) {
- return IS_NULL_RANGE;
+ if (lowerRange.length == 0 && upperRange.length == 0) {
+ // Need singleton to represent NULL range so it gets treated differently
+ // than an unbound RANGE.
+ return lowerInclusive && upperInclusive ? IS_NULL_RANGE : EVERYTHING_RANGE;
+ }
+ if (lowerRange.length != 0 && upperRange.length != 0) {
+ int cmp = Bytes.compareTo(lowerRange, upperRange);
+ if (cmp > 0 || (cmp == 0 && !(lowerInclusive && upperInclusive))) {
+ return EMPTY_RANGE;
+ }
+ }
+ return null;
+ }
+
+ public static KeyRange getKeyRange(byte[] lowerRange, boolean lowerInclusive,
+ byte[] upperRange, boolean upperInclusive) {
+ KeyRange range = getSingleton(lowerRange, lowerInclusive, upperRange, upperInclusive);
+ if (range != null) {
+ return range;
}
boolean unboundLower = false;
boolean unboundUpper = false;
@@ -142,20 +157,23 @@ public class KeyRange implements Writable {
unboundUpper = true;
}
- if (unboundLower && unboundUpper) {
- return EVERYTHING_RANGE;
- }
- if (!unboundLower && !unboundUpper) {
- int cmp = Bytes.compareTo(lowerRange, upperRange);
- if (cmp > 0 || (cmp == 0 && !(lowerInclusive && upperInclusive))) {
- return EMPTY_RANGE;
- }
- }
return new KeyRange(lowerRange, unboundLower ? false : lowerInclusive,
upperRange, unboundUpper ? false : upperInclusive);
}
- public KeyRange() {
+ public static KeyRange read(DataInput input) throws IOException {
+ KeyRange range = new KeyRange();
+ range.readFields(input);
+ // Translate to singleton after reading
+ KeyRange singletonRange = getSingleton(range.lowerRange, range.lowerInclusive, range.upperRange, range.upperInclusive);
+ if (singletonRange != null) {
+ return singletonRange;
+ }
+ // Otherwise, just keep the range we read
+ return range;
+ }
+
+ private KeyRange() {
this.lowerRange = DEGENERATE_KEY;
this.lowerInclusive = false;
this.upperRange = DEGENERATE_KEY;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ee813cee/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
index 912dd03..b6a2c85 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -375,7 +375,7 @@ public class ScanUtil {
* incrementing the key value itself, and thus bumping it up too much.
*/
boolean inclusiveUpper = range.isUpperInclusive() && bound == Bound.UPPER;
- boolean exclusiveLower = !range.isLowerInclusive() && bound == Bound.LOWER;
+ boolean exclusiveLower = !range.isLowerInclusive() && bound == Bound.LOWER && range != KeyRange.EVERYTHING_RANGE;
boolean exclusiveUpper = !range.isUpperInclusive() && bound == Bound.UPPER;
// If we are setting the upper bound of using inclusive single key, we remember
// to increment the key if we exit the loop after this iteration.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ee813cee/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java
index 7e68e25..898f778 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java
@@ -23,19 +23,17 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.List;
-import junit.framework.TestCase;
-
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryConstants;
-import org.apache.phoenix.schema.types.PChar;
-import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.PDatum;
-import org.apache.phoenix.schema.types.PVarchar;
import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder;
import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.types.PChar;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PVarchar;
import org.apache.phoenix.util.ByteUtil;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -45,6 +43,8 @@ import org.junit.runners.Parameterized.Parameters;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
+import junit.framework.TestCase;
+
//reset()
//filterAllRemaining() -> true indicates scan is over, false, keep going on.
//filterRowKey(byte[],int,int) -> true to drop this row, if false, we will also call
@@ -97,7 +97,6 @@ public class SkipScanFilterTest extends TestCase {
@Test
public void test() throws IOException {
- System.out.println("CNF: " + cnf + "\n" + "Expectations: " + expectations);
for (Expectation expectation : expectations) {
expectation.examine(skipper);
}
@@ -106,6 +105,39 @@ public class SkipScanFilterTest extends TestCase {
@Parameters(name="{0} {1} {2}")
public static Collection<Object> data() {
List<Object> testCases = Lists.newArrayList();
+ // Variable length tests
+ testCases.addAll(
+ foreach(new KeyRange[][]{{
+ PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("a"), true, Bytes.toBytes("a"), true),
+ PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("e"), true, Bytes.toBytes("e"), true),
+ PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("f"), true, Bytes.toBytes("f"), true),
+ },
+ {
+ PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("b"), true, Bytes.toBytes("b"), true),
+ },
+ {
+ KeyRange.EVERYTHING_RANGE,
+ },
+ {
+ PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("1"), true, Bytes.toBytes("1"), true),
+ }},
+ new int[4],
+ new Include(ByteUtil.concat(Bytes.toBytes("a"),QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("b"), QueryConstants.SEPARATOR_BYTE_ARRAY,
+ QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("1") ) ),
+ new SeekNext(ByteUtil.concat(Bytes.toBytes("e.f"),QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("b"), QueryConstants.SEPARATOR_BYTE_ARRAY,
+ QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("1") ),
+ ByteUtil.concat(Bytes.toBytes("f"),QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("b") )),
+ new Include(ByteUtil.concat(Bytes.toBytes("f"),QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("b"), QueryConstants.SEPARATOR_BYTE_ARRAY,
+ QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("1") ) ) )
+ );
+ // Fixed length tests
testCases.addAll(
foreach(new KeyRange[][]{{
PChar.INSTANCE.getKeyRange(Bytes.toBytes("abc"), true, Bytes.toBytes("def"), true),
@@ -124,7 +156,6 @@ public class SkipScanFilterTest extends TestCase {
PChar.INSTANCE.getKeyRange(Bytes.toBytes("AA"), true, Bytes.toBytes("AB"), false),
}},
new int[]{3,2,2,2,2},
- //new SeekNext("abcABABABAB", "abdAAAAAAAA"),
new SeekNext("defAAABABAB", "dzzAAAAAAAA"),
new Finished("xyyABABABAB"))
);
@@ -309,23 +340,6 @@ public class SkipScanFilterTest extends TestCase {
new SeekNext("dzzAB250", "dzzAB701"),
new Finished("zzzAA000"))
);
-// TODO variable length columns
-// testCases.addAll(
-// foreach(new KeyRange[][]{{
-// Char.INSTANCE.getKeyRange(Bytes.toBytes("apple"), true, Bytes.toBytes("lemon"), true),
-// Char.INSTANCE.getKeyRange(Bytes.toBytes("pear"), false, Bytes.toBytes("yam"), false),
-// },
-// {
-// Char.INSTANCE.getKeyRange(Bytes.toBytes("AB"), true, Bytes.toBytes("AX"), true),
-// Char.INSTANCE.getKeyRange(Bytes.toBytes("EA"), false, Bytes.toBytes("EZ"), false),
-// Char.INSTANCE.getKeyRange(Bytes.toBytes("PO"), true, Bytes.toBytes("PP"), false),
-// },
-// {
-// Char.INSTANCE.getKeyRange(Bytes.toBytes("100"), true, Bytes.toBytes("250"), false),
-// Char.INSTANCE.getKeyRange(Bytes.toBytes("700"), false, Bytes.toBytes("901"), false),
-// }},
-// new int[]{3,3})
-// );
return testCases;
}
@@ -378,6 +392,10 @@ public class SkipScanFilterTest extends TestCase {
this.rowkey = Bytes.toBytes(rowkey);
}
+ public Include(byte[] rowkey) {
+ this.rowkey = rowkey;
+ }
+
@SuppressWarnings("deprecation")
@Override public void examine(SkipScanFilter skipper) throws IOException {
KeyValue kv = KeyValue.createFirstOnRow(rowkey);
[19/50] [abbrv] phoenix git commit: PHOENIX-2492 Expose
PhoenixRecordWritable outside of spark
Posted by sa...@apache.org.
PHOENIX-2492 Expose PhoenixRecordWritable outside of spark
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/2f8d71db
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/2f8d71db
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/2f8d71db
Branch: refs/heads/4.x-HBase-1.0
Commit: 2f8d71db0e2757e2099c6e0d492b65364c7e409f
Parents: 7c76bba
Author: Nick Dimiduk <nd...@apache.org>
Authored: Fri Dec 4 16:04:25 2015 -0800
Committer: Nick Dimiduk <nd...@apache.org>
Committed: Tue Dec 22 16:12:30 2015 -0500
----------------------------------------------------------------------
.../mapreduce/PhoenixRecordWritable.java | 192 +++++++++++++++++++
1 file changed, 192 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/2f8d71db/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordWritable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordWritable.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordWritable.java
new file mode 100644
index 0000000..8d7d97a
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordWritable.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce;
+
+import org.apache.hadoop.mapreduce.lib.db.DBWritable;
+import org.apache.phoenix.schema.types.*;
+import org.apache.phoenix.util.ColumnInfo;
+import org.joda.time.DateTime;
+
+import java.sql.Array;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+public class PhoenixRecordWritable implements DBWritable {
+
+ private final List<Object> upsertValues = new ArrayList<>();
+ private final Map<String, Object> resultMap = new HashMap<>();
+ private List<ColumnInfo> columnMetaDataList;
+
+ /** For serialization; do not use. */
+ public PhoenixRecordWritable() {
+ this(new ArrayList<ColumnInfo>());
+ }
+
+ public PhoenixRecordWritable(List<ColumnInfo> columnMetaDataList) {
+ this.columnMetaDataList = columnMetaDataList;
+ }
+
+ /**
+ * Helper method to create a {@link Array} for a specific {@link PDataType}, and set it on
+ * the provided {@code stmt}.
+ */
+ private static void setArrayInStatement(PreparedStatement stmt, PDataType<?> type,
+ Object[] obj, int position) throws SQLException {
+ Array sqlArray = stmt.getConnection().createArrayOf(
+ PDataType.arrayBaseType(type).getSqlTypeName(), obj);
+ stmt.setArray(position, sqlArray);
+ }
+
+ private static Object[] primativeArrayToObjectArray(byte[] a) {
+ final Byte[] ret = new Byte[a.length];
+ for (int i = 0; i < a.length; i++) {
+ ret[i] = a[i];
+ }
+ return ret;
+ }
+
+ private static Object[] primativeArrayToObjectArray(short[] a) {
+ final Short[] ret = new Short[a.length];
+ for (int i = 0; i < a.length; i++) {
+ ret[i] = a[i];
+ }
+ return ret;
+ }
+
+ private static Object[] primativeArrayToObjectArray(int[] a) {
+ final Integer[] ret = new Integer[a.length];
+ for (int i = 0; i < a.length; i++) {
+ ret[i] = a[i];
+ }
+ return ret;
+ }
+
+ private static Object[] primativeArrayToObjectArray(float[] a) {
+ final Float[] ret = new Float[a.length];
+ for (int i = 0; i < a.length; i++) {
+ ret[i] = a[i];
+ }
+ return ret;
+ }
+
+ private static Object[] primativeArrayToObjectArray(double[] a) {
+ final Double[] ret = new Double[a.length];
+ for (int i = 0; i < a.length; i++) {
+ ret[i] = a[i];
+ }
+ return ret;
+ }
+
+ private static Object[] primativeArrayToObjectArray(long[] a) {
+ final Long[] ret = new Long[a.length];
+ for (int i = 0; i < a.length; i++) {
+ ret[i] = a[i];
+ }
+ return ret;
+ }
+
+ @Override public void write(PreparedStatement statement) throws SQLException {
+ // make sure we at least line up in size
+ if (upsertValues.size() != columnMetaDataList.size()) {
+ throw new UnsupportedOperationException("Provided " + upsertValues.size()
+ + " upsert values, but column metadata expects " + columnMetaDataList.size()
+ + " columns.");
+ }
+
+ // correlate each value (v) to a column type (c) and an index (i)
+ for (int i = 0; i < upsertValues.size(); i++) {
+ Object v = upsertValues.get(i);
+ ColumnInfo c = columnMetaDataList.get(i);
+
+ if (v == null) {
+ statement.setNull(i + 1, c.getSqlType());
+ continue;
+ }
+
+ // both Java and Joda dates used to work in 4.2.3, but now they must be java.sql.Date
+ // can override any other types here as needed
+ final Object finalObj;
+ final PDataType<?> finalType;
+ if (v instanceof DateTime) {
+ finalObj = new java.sql.Date(((DateTime) v).getMillis());
+ finalType = PDate.INSTANCE;
+ } else if (v instanceof java.util.Date) {
+ finalObj = new java.sql.Date(((java.util.Date) v).getTime());
+ finalType = PDate.INSTANCE;
+ } else {
+ finalObj = v;
+ finalType = c.getPDataType();
+ }
+
+ if (finalObj instanceof Object[]) {
+ setArrayInStatement(statement, finalType, (Object[]) finalObj, i + 1);
+ } else if (finalObj instanceof byte[]) {
+ // PVarbinary and PBinary are provided as byte[] but are treated as SQL objects
+ if (PDataType.equalsAny(finalType, PVarbinary.INSTANCE, PBinary.INSTANCE)) {
+ statement.setObject(i + 1, finalObj);
+ }
+ // otherwise set as array type
+ setArrayInStatement(statement, finalType, primativeArrayToObjectArray((byte[]) finalObj), i + 1);
+ } else if (finalObj instanceof short[]) {
+ setArrayInStatement(statement, finalType, primativeArrayToObjectArray((short[]) finalObj), i + 1);
+ } else if (finalObj instanceof int[]) {
+ setArrayInStatement(statement, finalType, primativeArrayToObjectArray((int[]) finalObj), i + 1);
+ } else if (finalObj instanceof long[]) {
+ setArrayInStatement(statement, finalType, primativeArrayToObjectArray((long[]) finalObj), i + 1);
+ } else if (finalObj instanceof float[]) {
+ setArrayInStatement(statement, finalType, primativeArrayToObjectArray((float[]) finalObj), i + 1);
+ } else if (finalObj instanceof double[]) {
+ setArrayInStatement(statement, finalType, primativeArrayToObjectArray((double[]) finalObj), i + 1);
+ } else {
+ statement.setObject(i + 1, finalObj);
+ }
+ }
+ }
+
+ @Override public void readFields(ResultSet resultSet) throws SQLException {
+ ResultSetMetaData metaData = resultSet.getMetaData();
+ for (int i = 1; i <= metaData.getColumnCount(); i++) {
+ // return the contents of a PhoenixArray, if necessary
+ Object value = resultSet.getObject(i);
+ if (value instanceof PhoenixArray) {
+ value = ((PhoenixArray) value).getArray();
+ }
+
+ // put a (ColumnLabel -> value) entry into the result map
+ resultMap.put(metaData.getColumnLabel(i), value);
+ }
+ }
+
+ /** Append an object to the list of values to upsert. */
+ public void add(Object value) {
+ upsertValues.add(value);
+ }
+
+ /** @return an immutable view on the {@link ResultSet} content. */
+ public Map<String, Object> getResultMap() {
+ return Collections.unmodifiableMap(resultMap);
+ }
+}
[30/50] [abbrv] phoenix git commit: PHOENIX-2551 Remove Jdbc7Shim as
it's no longer necessary
Posted by sa...@apache.org.
PHOENIX-2551 Remove Jdbc7Shim as it's no longer necessary
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/708a8005
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/708a8005
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/708a8005
Branch: refs/heads/4.x-HBase-1.0
Commit: 708a8005320127aecd160ade67be471b2fc68130
Parents: 267715a
Author: James Taylor <jt...@salesforce.com>
Authored: Wed Dec 30 09:19:41 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Dec 30 18:14:25 2015 -0800
----------------------------------------------------------------------
.../phoenix/trace/DelegateConnection.java | 324 ++++++++++++++++++
.../phoenix/trace/DelegatingConnection.java | 328 -------------------
.../phoenix/trace/PhoenixTracingEndToEndIT.java | 3 +-
.../java/org/apache/phoenix/jdbc/Jdbc7Shim.java | 64 ----
.../apache/phoenix/jdbc/PhoenixConnection.java | 2 +-
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 2 +-
.../phoenix/jdbc/PhoenixEmbeddedDriver.java | 3 +-
.../apache/phoenix/jdbc/PhoenixResultSet.java | 2 +-
.../apache/phoenix/jdbc/PhoenixStatement.java | 2 +-
9 files changed, 330 insertions(+), 400 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/708a8005/phoenix-core/src/it/java/org/apache/phoenix/trace/DelegateConnection.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/trace/DelegateConnection.java b/phoenix-core/src/it/java/org/apache/phoenix/trace/DelegateConnection.java
new file mode 100644
index 0000000..8fff469
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/trace/DelegateConnection.java
@@ -0,0 +1,324 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.trace;
+
+import java.sql.Array;
+import java.sql.Blob;
+import java.sql.CallableStatement;
+import java.sql.Clob;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.NClob;
+import java.sql.PreparedStatement;
+import java.sql.SQLClientInfoException;
+import java.sql.SQLException;
+import java.sql.SQLWarning;
+import java.sql.SQLXML;
+import java.sql.Savepoint;
+import java.sql.Statement;
+import java.sql.Struct;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.Executor;
+
+/**
+ * Simple {@link Connection} that just delegates to an underlying {@link Connection}.
+ */
+public class DelegateConnection implements Connection {
+
+ @Override
+ public <T> T unwrap(Class<T> iface) throws SQLException {
+ return conn.unwrap(iface);
+ }
+
+ @Override
+ public boolean isWrapperFor(Class<?> iface) throws SQLException {
+ return conn.isWrapperFor(iface);
+ }
+
+ @Override
+ public Statement createStatement() throws SQLException {
+ return conn.createStatement();
+ }
+
+ @Override
+ public PreparedStatement prepareStatement(String sql) throws SQLException {
+ return conn.prepareStatement(sql);
+ }
+
+ @Override
+ public CallableStatement prepareCall(String sql) throws SQLException {
+ return conn.prepareCall(sql);
+ }
+
+ @Override
+ public String nativeSQL(String sql) throws SQLException {
+ return conn.nativeSQL(sql);
+ }
+
+ @Override
+ public void setAutoCommit(boolean autoCommit) throws SQLException {
+ conn.setAutoCommit(autoCommit);
+ }
+
+ @Override
+ public boolean getAutoCommit() throws SQLException {
+ return conn.getAutoCommit();
+ }
+
+ @Override
+ public void commit() throws SQLException {
+ conn.commit();
+ }
+
+ @Override
+ public void rollback() throws SQLException {
+ conn.rollback();
+ }
+
+ @Override
+ public void close() throws SQLException {
+ conn.close();
+ }
+
+ @Override
+ public boolean isClosed() throws SQLException {
+ return conn.isClosed();
+ }
+
+ @Override
+ public DatabaseMetaData getMetaData() throws SQLException {
+ return conn.getMetaData();
+ }
+
+ @Override
+ public void setReadOnly(boolean readOnly) throws SQLException {
+ conn.setReadOnly(readOnly);
+ }
+
+ @Override
+ public boolean isReadOnly() throws SQLException {
+ return conn.isReadOnly();
+ }
+
+ @Override
+ public void setCatalog(String catalog) throws SQLException {
+ conn.setCatalog(catalog);
+ }
+
+ @Override
+ public String getCatalog() throws SQLException {
+ return conn.getCatalog();
+ }
+
+ @Override
+ public void setTransactionIsolation(int level) throws SQLException {
+ conn.setTransactionIsolation(level);
+ }
+
+ @Override
+ public int getTransactionIsolation() throws SQLException {
+ return conn.getTransactionIsolation();
+ }
+
+ @Override
+ public SQLWarning getWarnings() throws SQLException {
+ return conn.getWarnings();
+ }
+
+ @Override
+ public void clearWarnings() throws SQLException {
+ conn.clearWarnings();
+ }
+
+ @Override
+ public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException {
+ return conn.createStatement(resultSetType, resultSetConcurrency);
+ }
+
+ @Override
+ public PreparedStatement
+ prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException {
+ return conn.prepareStatement(sql, resultSetType, resultSetConcurrency);
+ }
+
+ @Override
+ public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency)
+ throws SQLException {
+ return conn.prepareCall(sql, resultSetType, resultSetConcurrency);
+ }
+
+ @Override
+ public Map<String, Class<?>> getTypeMap() throws SQLException {
+ return conn.getTypeMap();
+ }
+
+ @Override
+ public void setTypeMap(Map<String, Class<?>> map) throws SQLException {
+ conn.setTypeMap(map);
+ }
+
+ @Override
+ public void setHoldability(int holdability) throws SQLException {
+ conn.setHoldability(holdability);
+ }
+
+ @Override
+ public int getHoldability() throws SQLException {
+ return conn.getHoldability();
+ }
+
+ @Override
+ public Savepoint setSavepoint() throws SQLException {
+ return conn.setSavepoint();
+ }
+
+ @Override
+ public Savepoint setSavepoint(String name) throws SQLException {
+ return conn.setSavepoint(name);
+ }
+
+ @Override
+ public void rollback(Savepoint savepoint) throws SQLException {
+ conn.rollback(savepoint);
+ }
+
+ @Override
+ public void releaseSavepoint(Savepoint savepoint) throws SQLException {
+ conn.releaseSavepoint(savepoint);
+ }
+
+ @Override
+ public Statement createStatement(int resultSetType, int resultSetConcurrency,
+ int resultSetHoldability) throws SQLException {
+ return conn.createStatement(resultSetType, resultSetConcurrency, resultSetHoldability);
+ }
+
+ @Override
+ public PreparedStatement prepareStatement(String sql, int resultSetType,
+ int resultSetConcurrency, int resultSetHoldability) throws SQLException {
+ return conn.prepareStatement(sql, resultSetType, resultSetConcurrency, resultSetHoldability);
+ }
+
+ @Override
+ public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency,
+ int resultSetHoldability) throws SQLException {
+ return conn.prepareCall(sql, resultSetType, resultSetConcurrency, resultSetHoldability);
+ }
+
+ @Override
+ public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException {
+ return conn.prepareStatement(sql, autoGeneratedKeys);
+ }
+
+ @Override
+ public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException {
+ return conn.prepareStatement(sql, columnIndexes);
+ }
+
+ @Override
+ public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException {
+ return conn.prepareStatement(sql, columnNames);
+ }
+
+ @Override
+ public Clob createClob() throws SQLException {
+ return conn.createClob();
+ }
+
+ @Override
+ public Blob createBlob() throws SQLException {
+ return conn.createBlob();
+ }
+
+ @Override
+ public NClob createNClob() throws SQLException {
+ return conn.createNClob();
+ }
+
+ @Override
+ public SQLXML createSQLXML() throws SQLException {
+ return conn.createSQLXML();
+ }
+
+ @Override
+ public boolean isValid(int timeout) throws SQLException {
+ return conn.isValid(timeout);
+ }
+
+ @Override
+ public void setClientInfo(String name, String value) throws SQLClientInfoException {
+ conn.setClientInfo(name, value);
+ }
+
+ @Override
+ public void setClientInfo(Properties properties) throws SQLClientInfoException {
+ conn.setClientInfo(properties);
+ }
+
+ @Override
+ public String getClientInfo(String name) throws SQLException {
+ return conn.getClientInfo(name);
+ }
+
+ @Override
+ public Properties getClientInfo() throws SQLException {
+ return conn.getClientInfo();
+ }
+
+ @Override
+ public Array createArrayOf(String typeName, Object[] elements) throws SQLException {
+ return conn.createArrayOf(typeName, elements);
+ }
+
+ @Override
+ public Struct createStruct(String typeName, Object[] attributes) throws SQLException {
+ return conn.createStruct(typeName, attributes);
+ }
+
+ private Connection conn;
+
+ public DelegateConnection(Connection conn) {
+ this.conn = conn;
+ }
+
+ @Override
+ public void setSchema(String schema) throws SQLException {
+ conn.setSchema(schema);
+ }
+
+ @Override
+ public String getSchema() throws SQLException {
+ return conn.getSchema();
+ }
+
+ @Override
+ public void abort(Executor executor) throws SQLException {
+ conn.abort(executor);
+ }
+
+ @Override
+ public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException {
+ conn.setNetworkTimeout(executor, milliseconds);
+ }
+
+ @Override
+ public int getNetworkTimeout() throws SQLException {
+ return conn.getNetworkTimeout();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/708a8005/phoenix-core/src/it/java/org/apache/phoenix/trace/DelegatingConnection.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/trace/DelegatingConnection.java b/phoenix-core/src/it/java/org/apache/phoenix/trace/DelegatingConnection.java
deleted file mode 100644
index 261522d..0000000
--- a/phoenix-core/src/it/java/org/apache/phoenix/trace/DelegatingConnection.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.trace;
-
-import java.sql.Array;
-import java.sql.Blob;
-import java.sql.CallableStatement;
-import java.sql.Clob;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.NClob;
-import java.sql.PreparedStatement;
-import java.sql.SQLClientInfoException;
-import java.sql.SQLException;
-import java.sql.SQLWarning;
-import java.sql.SQLXML;
-import java.sql.Savepoint;
-import java.sql.Statement;
-import java.sql.Struct;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.Executor;
-
-import org.apache.phoenix.jdbc.Jdbc7Shim;
-
-/**
- * Simple {@link Connection} that just delegates to an underlying {@link Connection}.
- * @param <D> delegate type that is both a {@link Connection} and a {@link Jdbc7Shim#Connection}
- */
-public class DelegatingConnection<D extends Connection & Jdbc7Shim.Connection> implements
- Connection, Jdbc7Shim.Connection {
-
- @Override
- public <T> T unwrap(Class<T> iface) throws SQLException {
- return conn.unwrap(iface);
- }
-
- @Override
- public boolean isWrapperFor(Class<?> iface) throws SQLException {
- return conn.isWrapperFor(iface);
- }
-
- @Override
- public Statement createStatement() throws SQLException {
- return conn.createStatement();
- }
-
- @Override
- public PreparedStatement prepareStatement(String sql) throws SQLException {
- return conn.prepareStatement(sql);
- }
-
- @Override
- public CallableStatement prepareCall(String sql) throws SQLException {
- return conn.prepareCall(sql);
- }
-
- @Override
- public String nativeSQL(String sql) throws SQLException {
- return conn.nativeSQL(sql);
- }
-
- @Override
- public void setAutoCommit(boolean autoCommit) throws SQLException {
- conn.setAutoCommit(autoCommit);
- }
-
- @Override
- public boolean getAutoCommit() throws SQLException {
- return conn.getAutoCommit();
- }
-
- @Override
- public void commit() throws SQLException {
- conn.commit();
- }
-
- @Override
- public void rollback() throws SQLException {
- conn.rollback();
- }
-
- @Override
- public void close() throws SQLException {
- conn.close();
- }
-
- @Override
- public boolean isClosed() throws SQLException {
- return conn.isClosed();
- }
-
- @Override
- public DatabaseMetaData getMetaData() throws SQLException {
- return conn.getMetaData();
- }
-
- @Override
- public void setReadOnly(boolean readOnly) throws SQLException {
- conn.setReadOnly(readOnly);
- }
-
- @Override
- public boolean isReadOnly() throws SQLException {
- return conn.isReadOnly();
- }
-
- @Override
- public void setCatalog(String catalog) throws SQLException {
- conn.setCatalog(catalog);
- }
-
- @Override
- public String getCatalog() throws SQLException {
- return conn.getCatalog();
- }
-
- @Override
- public void setTransactionIsolation(int level) throws SQLException {
- conn.setTransactionIsolation(level);
- }
-
- @Override
- public int getTransactionIsolation() throws SQLException {
- return conn.getTransactionIsolation();
- }
-
- @Override
- public SQLWarning getWarnings() throws SQLException {
- return conn.getWarnings();
- }
-
- @Override
- public void clearWarnings() throws SQLException {
- conn.clearWarnings();
- }
-
- @Override
- public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException {
- return conn.createStatement(resultSetType, resultSetConcurrency);
- }
-
- @Override
- public PreparedStatement
- prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException {
- return conn.prepareStatement(sql, resultSetType, resultSetConcurrency);
- }
-
- @Override
- public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency)
- throws SQLException {
- return conn.prepareCall(sql, resultSetType, resultSetConcurrency);
- }
-
- @Override
- public Map<String, Class<?>> getTypeMap() throws SQLException {
- return conn.getTypeMap();
- }
-
- @Override
- public void setTypeMap(Map<String, Class<?>> map) throws SQLException {
- conn.setTypeMap(map);
- }
-
- @Override
- public void setHoldability(int holdability) throws SQLException {
- conn.setHoldability(holdability);
- }
-
- @Override
- public int getHoldability() throws SQLException {
- return conn.getHoldability();
- }
-
- @Override
- public Savepoint setSavepoint() throws SQLException {
- return conn.setSavepoint();
- }
-
- @Override
- public Savepoint setSavepoint(String name) throws SQLException {
- return conn.setSavepoint(name);
- }
-
- @Override
- public void rollback(Savepoint savepoint) throws SQLException {
- conn.rollback(savepoint);
- }
-
- @Override
- public void releaseSavepoint(Savepoint savepoint) throws SQLException {
- conn.releaseSavepoint(savepoint);
- }
-
- @Override
- public Statement createStatement(int resultSetType, int resultSetConcurrency,
- int resultSetHoldability) throws SQLException {
- return conn.createStatement(resultSetType, resultSetConcurrency, resultSetHoldability);
- }
-
- @Override
- public PreparedStatement prepareStatement(String sql, int resultSetType,
- int resultSetConcurrency, int resultSetHoldability) throws SQLException {
- return conn.prepareStatement(sql, resultSetType, resultSetConcurrency, resultSetHoldability);
- }
-
- @Override
- public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency,
- int resultSetHoldability) throws SQLException {
- return conn.prepareCall(sql, resultSetType, resultSetConcurrency, resultSetHoldability);
- }
-
- @Override
- public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException {
- return conn.prepareStatement(sql, autoGeneratedKeys);
- }
-
- @Override
- public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException {
- return conn.prepareStatement(sql, columnIndexes);
- }
-
- @Override
- public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException {
- return conn.prepareStatement(sql, columnNames);
- }
-
- @Override
- public Clob createClob() throws SQLException {
- return conn.createClob();
- }
-
- @Override
- public Blob createBlob() throws SQLException {
- return conn.createBlob();
- }
-
- @Override
- public NClob createNClob() throws SQLException {
- return conn.createNClob();
- }
-
- @Override
- public SQLXML createSQLXML() throws SQLException {
- return conn.createSQLXML();
- }
-
- @Override
- public boolean isValid(int timeout) throws SQLException {
- return conn.isValid(timeout);
- }
-
- @Override
- public void setClientInfo(String name, String value) throws SQLClientInfoException {
- conn.setClientInfo(name, value);
- }
-
- @Override
- public void setClientInfo(Properties properties) throws SQLClientInfoException {
- conn.setClientInfo(properties);
- }
-
- @Override
- public String getClientInfo(String name) throws SQLException {
- return conn.getClientInfo(name);
- }
-
- @Override
- public Properties getClientInfo() throws SQLException {
- return conn.getClientInfo();
- }
-
- @Override
- public Array createArrayOf(String typeName, Object[] elements) throws SQLException {
- return conn.createArrayOf(typeName, elements);
- }
-
- @Override
- public Struct createStruct(String typeName, Object[] attributes) throws SQLException {
- return conn.createStruct(typeName, attributes);
- }
-
- private D conn;
-
- public DelegatingConnection(D conn) {
- this.conn = conn;
- }
-
- @Override
- public void setSchema(String schema) throws SQLException {
- conn.setSchema(schema);
- }
-
- @Override
- public String getSchema() throws SQLException {
- return conn.getSchema();
- }
-
- @Override
- public void abort(Executor executor) throws SQLException {
- conn.abort(executor);
- }
-
- @Override
- public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException {
- conn.setNetworkTimeout(executor, milliseconds);
- }
-
- @Override
- public int getNetworkTimeout() throws SQLException {
- return conn.getNetworkTimeout();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/708a8005/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java b/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java
index 8febfff..2cd795d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java
@@ -496,10 +496,9 @@ public class PhoenixTracingEndToEndIT extends BaseTracingTestIT {
}
}
- private static class CountDownConnection extends DelegatingConnection {
+ private static class CountDownConnection extends DelegateConnection {
private CountDownLatch commit;
- @SuppressWarnings("unchecked")
public CountDownConnection(Connection conn, CountDownLatch commit) {
super(conn);
this.commit = commit;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/708a8005/phoenix-core/src/main/java/org/apache/phoenix/jdbc/Jdbc7Shim.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/Jdbc7Shim.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/Jdbc7Shim.java
deleted file mode 100644
index f51da30..0000000
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/Jdbc7Shim.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.jdbc;
-
-import java.sql.SQLException;
-import java.sql.SQLFeatureNotSupportedException;
-import java.util.concurrent.Executor;
-import java.util.logging.Logger;
-
-/**
- * Interfaces to be implemented by classes that need to be "JDK7" compliant,
- * but also run in JDK6
- */
-public final class Jdbc7Shim {
-
- public interface Statement { // Note: do not extend "regular" statement or else eclipse 3.7 complains
- void closeOnCompletion() throws SQLException;
- boolean isCloseOnCompletion() throws SQLException;
- }
-
- public interface CallableStatement extends Statement {
- public <T> T getObject(int columnIndex, Class<T> type) throws SQLException;
- public <T> T getObject(String columnLabel, Class<T> type) throws SQLException;
- }
-
- public interface Connection {
- void setSchema(String schema) throws SQLException;
- String getSchema() throws SQLException;
- void abort(Executor executor) throws SQLException;
- void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException;
- int getNetworkTimeout() throws SQLException;
- }
-
- public interface ResultSet {
- public <T> T getObject(int columnIndex, Class<T> type) throws SQLException;
- public <T> T getObject(String columnLabel, Class<T> type) throws SQLException;
- }
-
- public interface DatabaseMetaData {
- java.sql.ResultSet getPseudoColumns(String catalog, String schemaPattern,
- String tableNamePattern, String columnNamePattern)
- throws SQLException;
- boolean generatedKeyAlwaysReturned() throws SQLException;
- }
-
- public interface Driver {
- public Logger getParentLogger() throws SQLFeatureNotSupportedException;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/708a8005/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
index 6a25184..2d8810f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
@@ -123,7 +123,7 @@ import co.cask.tephra.TransactionContext;
*
* @since 0.1
*/
-public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jdbc7Shim.Connection, MetaDataMutated{
+public class PhoenixConnection implements Connection, MetaDataMutated {
private final String url;
private final ConnectionQueryServices services;
private final Properties info;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/708a8005/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index 940ca52..87c02d0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -98,7 +98,7 @@ import com.google.common.collect.Lists;
*
* @since 0.1
*/
-public class PhoenixDatabaseMetaData implements DatabaseMetaData, org.apache.phoenix.jdbc.Jdbc7Shim.DatabaseMetaData {
+public class PhoenixDatabaseMetaData implements DatabaseMetaData {
public static final int INDEX_NAME_INDEX = 4; // Shared with FAMILY_NAME_INDEX
public static final int FAMILY_NAME_INDEX = 4;
public static final int COLUMN_NAME_INDEX = 3;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/708a8005/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java
index 8ee9dbd..c49bf37 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java
@@ -31,7 +31,6 @@ import java.util.logging.Logger;
import javax.annotation.concurrent.Immutable;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.exception.SQLExceptionInfo;
@@ -55,7 +54,7 @@ import com.google.common.collect.Maps;
* @since 0.1
*/
@Immutable
-public abstract class PhoenixEmbeddedDriver implements Driver, org.apache.phoenix.jdbc.Jdbc7Shim.Driver, SQLCloseable {
+public abstract class PhoenixEmbeddedDriver implements Driver, SQLCloseable {
/**
* The protocol for Phoenix Network Client
*/
http://git-wip-us.apache.org/repos/asf/phoenix/blob/708a8005/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java
index c1bbe81..a3ce1a1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java
@@ -100,7 +100,7 @@ import com.google.common.annotations.VisibleForTesting;
*
* @since 0.1
*/
-public class PhoenixResultSet implements ResultSet, SQLCloseable, org.apache.phoenix.jdbc.Jdbc7Shim.ResultSet {
+public class PhoenixResultSet implements ResultSet, SQLCloseable {
private static final Log LOG = LogFactory.getLog(PhoenixResultSet.class);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/708a8005/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index cb44c3f..7e8969b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -180,7 +180,7 @@ import com.google.common.math.IntMath;
*
* @since 0.1
*/
-public class PhoenixStatement implements Statement, SQLCloseable, org.apache.phoenix.jdbc.Jdbc7Shim.Statement {
+public class PhoenixStatement implements Statement, SQLCloseable {
private static final Logger logger = LoggerFactory.getLogger(PhoenixStatement.class);
[05/50] [abbrv] phoenix git commit: PHOENIX-2519 Prevent RPC for
SYSTEM tables when querying
Posted by sa...@apache.org.
PHOENIX-2519 Prevent RPC for SYSTEM tables when querying
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/72f80bd9
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/72f80bd9
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/72f80bd9
Branch: refs/heads/4.x-HBase-1.0
Commit: 72f80bd9d1452d0fca72aa5a3fcbe3996df8ca52
Parents: 8726efd
Author: James Taylor <jt...@salesforce.com>
Authored: Sun Dec 13 14:31:00 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Dec 14 11:52:00 2015 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/rpc/UpdateCacheIT.java | 46 +++++++++++++++-----
.../phoenix/rpc/UpdateCacheWithScnIT.java | 2 +-
.../apache/phoenix/execute/BaseQueryPlan.java | 12 ++++-
3 files changed, 47 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/72f80bd9/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
index 3b3c6c7..208af94 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
@@ -30,6 +30,7 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.sql.Connection;
+import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -78,26 +79,49 @@ public class UpdateCacheIT extends BaseHBaseManagedTimeIT {
ensureTableCreated(getUrl(), TRANSACTIONAL_DATA_TABLE);
}
+ private static void setupSystemTable(Long scn) throws SQLException {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ if (scn != null) {
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(scn));
+ }
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.createStatement().execute(
+ "create table " + QueryConstants.SYSTEM_SCHEMA_NAME + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE + TEST_TABLE_SCHEMA);
+ }
+ }
+
@Test
public void testUpdateCacheForTxnTable() throws Exception {
- helpTestUpdateCache(true, null);
+ helpTestUpdateCache(true, false, null);
}
@Test
public void testUpdateCacheForNonTxnTable() throws Exception {
- helpTestUpdateCache(false, null);
+ helpTestUpdateCache(false, false, null);
}
- public static void helpTestUpdateCache(boolean isTransactional, Long scn) throws Exception {
+ @Test
+ public void testUpdateCacheForNonTxnSystemTable() throws Exception {
+ helpTestUpdateCache(false, true, null);
+ }
+
+ public static void helpTestUpdateCache(boolean isTransactional, boolean isSystem, Long scn) throws Exception {
String tableName = isTransactional ? TRANSACTIONAL_DATA_TABLE : MUTABLE_INDEX_DATA_TABLE;
- String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + tableName;
+ String schemaName;
+ if (isSystem) {
+ setupSystemTable(scn);
+ schemaName = QueryConstants.SYSTEM_SCHEMA_NAME;
+ } else {
+ schemaName = INDEX_DATA_SCHEMA;
+ }
+ String fullTableName = schemaName + QueryConstants.NAME_SEPARATOR + tableName;
String selectSql = "SELECT * FROM "+fullTableName;
// use a spyed ConnectionQueryServices so we can verify calls to getTable
ConnectionQueryServices connectionQueryServices = Mockito.spy(driver.getConnectionQueryServices(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)));
Properties props = new Properties();
props.putAll(PhoenixEmbeddedDriver.DEFFAULT_PROPS.asMap());
if (scn!=null) {
- props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(scn));
+ props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(scn+10));
}
Connection conn = connectionQueryServices.connect(getUrl(), props);
try {
@@ -112,13 +136,14 @@ public class UpdateCacheIT extends BaseHBaseManagedTimeIT {
TestUtil.setRowKeyColumns(stmt, 3);
stmt.execute();
conn.commit();
- // verify only one rpc to fetch table metadata,
- verify(connectionQueryServices).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(INDEX_DATA_SCHEMA)), eq(PVarchar.INSTANCE.toBytes(tableName)), anyLong(), anyLong());
+ int numUpsertRpcs = isSystem ? 0 : 1;
+ // verify only 0 or 1 rpc to fetch table metadata,
+ verify(connectionQueryServices, times(numUpsertRpcs)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)), anyLong(), anyLong());
reset(connectionQueryServices);
if (scn!=null) {
// advance scn so that we can see the data we just upserted
- props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(scn+2));
+ props.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(scn+20));
conn = connectionQueryServices.connect(getUrl(), props);
}
@@ -143,8 +168,9 @@ public class UpdateCacheIT extends BaseHBaseManagedTimeIT {
// for non-transactional tables without a scn : verify one rpc to getTable occurs *per* query
// for non-transactional tables with a scn : verify *only* one rpc occurs
// for transactional tables : verify *only* one rpc occurs
- int numRpcs = isTransactional || scn!=null ? 1 : 3;
- verify(connectionQueryServices, times(numRpcs)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(INDEX_DATA_SCHEMA)), eq(PVarchar.INSTANCE.toBytes(tableName)), anyLong(), anyLong());
+ // for non-transactional, system tables : verify non rpc occurs
+ int numRpcs = isSystem ? 0 : (isTransactional || scn!=null ? 1 : 3);
+ verify(connectionQueryServices, times(numRpcs)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)), anyLong(), anyLong());
}
finally {
conn.close();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/72f80bd9/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java
index dbc7fd1..5ff2fb0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java
@@ -35,7 +35,7 @@ public class UpdateCacheWithScnIT extends BaseClientManagedTimeIT {
@Test
public void testUpdateCacheWithScn() throws Exception {
- UpdateCacheIT.helpTestUpdateCache(false, ts+2);
+ UpdateCacheIT.helpTestUpdateCache(false, false, ts+2);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/72f80bd9/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
index 91c2f19..7515642 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
@@ -28,6 +28,7 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.TimeRange;
@@ -57,6 +58,7 @@ import org.apache.phoenix.parse.FilterableStatement;
import org.apache.phoenix.parse.HintNode.Hint;
import org.apache.phoenix.parse.ParseNodeFactory;
import org.apache.phoenix.parse.TableName;
+import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.KeyValueSchema;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PName;
@@ -203,7 +205,8 @@ public abstract class BaseQueryPlan implements QueryPlan {
// Set miscellaneous scan attributes. This is the last chance to set them before we
// clone the scan for each parallelized chunk.
Scan scan = context.getScan();
- PTable table = context.getCurrentTable().getTable();
+ TableRef tableRef = context.getCurrentTable();
+ PTable table = tableRef.getTable();
if (dynamicFilter != null) {
WhereCompiler.compile(context, statement, null, Collections.singletonList(dynamicFilter), false, null);
@@ -232,7 +235,12 @@ public abstract class BaseQueryPlan implements QueryPlan {
TimeRange scanTimeRange = scan.getTimeRange();
Long scn = connection.getSCN();
if (scn == null) {
- scn = context.getCurrentTime();
+ // If we haven't resolved the time at the beginning of compilation, don't
+ // force the lookup on the server, but use HConstants.LATEST_TIMESTAMP instead.
+ scn = tableRef.getTimeStamp();
+ if (scn == QueryConstants.UNSET_TIMESTAMP) {
+ scn = HConstants.LATEST_TIMESTAMP;
+ }
}
try {
TimeRange timeRangeToUse = ScanUtil.intersectTimeRange(rowTimestampRange, scanTimeRange, scn);
[16/50] [abbrv] phoenix git commit: PHOENIX-2470 Skip overlapped scan
left after StaleRegionBoundaryCacheException(Ankit Singhal)
Posted by sa...@apache.org.
PHOENIX-2470 Skip overlapped scan left after StaleRegionBoundaryCacheException(Ankit Singhal)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/95cff2d4
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/95cff2d4
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/95cff2d4
Branch: refs/heads/4.x-HBase-1.0
Commit: 95cff2d49e9c83977852f89c806ecbf55309caa0
Parents: 785f487
Author: Rajeshbabu Chintaguntla <ra...@apache.org>
Authored: Mon Dec 21 08:07:28 2015 +0530
Committer: Rajeshbabu Chintaguntla <ra...@apache.org>
Committed: Mon Dec 21 08:07:28 2015 +0530
----------------------------------------------------------------------
.../phoenix/iterate/BaseResultIterators.java | 42 +++++++++++++++++---
1 file changed, 37 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/95cff2d4/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 6ad57c3..19c60bf 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -543,13 +543,31 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
// contain them all (for example if work was rejected from the queue)
Queue<PeekingResultIterator> allIterators = new ConcurrentLinkedQueue<>();
List<PeekingResultIterator> iterators = new ArrayList<PeekingResultIterator>(numScans);
+ ScanWrapper previousScan = new ScanWrapper(null);
return getIterators(scans, services, isLocalIndex, allIterators, iterators, isReverse, maxQueryEndTime,
- splits.size());
+ splits.size(), previousScan);
}
- private List<PeekingResultIterator> getIterators(List<List<Scan>> scan, ConnectionQueryServices services, boolean isLocalIndex,
- Queue<PeekingResultIterator> allIterators, List<PeekingResultIterator> iterators, boolean isReverse, long maxQueryEndTime, int splitSize)
- throws SQLException {
+ class ScanWrapper {
+ Scan scan;
+
+ public Scan getScan() {
+ return scan;
+ }
+
+ public void setScan(Scan scan) {
+ this.scan = scan;
+ }
+
+ public ScanWrapper(Scan scan) {
+ this.scan = scan;
+ }
+
+ }
+
+ private List<PeekingResultIterator> getIterators(List<List<Scan>> scan, ConnectionQueryServices services,
+ boolean isLocalIndex, Queue<PeekingResultIterator> allIterators, List<PeekingResultIterator> iterators,
+ boolean isReverse, long maxQueryEndTime, int splitSize, ScanWrapper previousScan) throws SQLException {
boolean success = false;
final List<List<Pair<Scan,Future<PeekingResultIterator>>>> futures = Lists.newArrayListWithExpectedSize(splitSize);
allFutures.add(futures);
@@ -566,8 +584,22 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
if (timeOutForScan < 0) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.OPERATION_TIMED_OUT).setMessage(". Query couldn't be completed in the alloted time: " + queryTimeOut + " ms").build().buildException();
}
+ if (isLocalIndex && previousScan != null && previousScan.getScan() != null
+ && ((!isReverse && Bytes.compareTo(scanPair.getFirst().getStartRow(),
+ previousScan.getScan().getStopRow()) < 0)
+ || (isReverse && Bytes.compareTo(scanPair.getFirst().getStartRow(),
+ previousScan.getScan().getStopRow()) > 0)
+ || (scanPair.getFirst().getAttribute(EXPECTED_UPPER_REGION_KEY) != null
+ && previousScan.getScan().getAttribute(EXPECTED_UPPER_REGION_KEY) != null
+ && Bytes.compareTo(scanPair.getFirst().getAttribute(EXPECTED_UPPER_REGION_KEY),
+ previousScan.getScan()
+ .getAttribute(EXPECTED_UPPER_REGION_KEY)) == 0))) {
+
+ continue;
+ }
PeekingResultIterator iterator = scanPair.getSecond().get(timeOutForScan, TimeUnit.MILLISECONDS);
concatIterators.add(iterator);
+ previousScan.setScan(scanPair.getFirst());
} catch (ExecutionException e) {
try { // Rethrow as SQLException
throw ServerUtil.parseServerException(e);
@@ -590,7 +622,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
addIterator(iterators, concatIterators);
concatIterators = Lists.newArrayList();
getIterators(newNestedScans, services, isLocalIndex, allIterators, iterators, isReverse,
- maxQueryEndTime, newNestedScans.size());
+ maxQueryEndTime, newNestedScans.size(), previousScan);
}
}
}
[17/50] [abbrv] phoenix git commit: PHOENIX-1312 Do not always
project the empty column family (Ram)
Posted by sa...@apache.org.
PHOENIX-1312 Do not always project the empty column family (Ram)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b35cb98c
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b35cb98c
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b35cb98c
Branch: refs/heads/4.x-HBase-1.0
Commit: b35cb98caece0183ce4dc29fcae4bfc80dbdc504
Parents: 95cff2d
Author: ramkrishna <ra...@gmail.com>
Authored: Mon Dec 21 13:35:40 2015 +0530
Committer: ramkrishna <ra...@gmail.com>
Committed: Mon Dec 21 13:40:05 2015 +0530
----------------------------------------------------------------------
.../phoenix/end2end/MultiCfQueryExecIT.java | 69 +++++++++++++++-
.../phoenix/compile/ProjectionCompiler.java | 16 ++--
.../apache/phoenix/compile/QueryCompiler.java | 2 +-
.../apache/phoenix/compile/RowProjector.java | 18 +++--
.../phoenix/compile/StatementContext.java | 2 +-
.../phoenix/iterate/BaseResultIterators.java | 84 ++++++++++++--------
.../phoenix/compile/QueryCompilerTest.java | 33 +++++++-
.../java/org/apache/phoenix/util/TestUtil.java | 12 +--
8 files changed, 179 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b35cb98c/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java
index 2edf189..f5566ce 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java
@@ -172,17 +172,80 @@ public class MultiCfQueryExecIT extends BaseOwnClusterClientManagedTimeIT {
assertFalse(rs.next());
// Use E column family. Since the column family with the empty key value (the first one, A)
// is always added to the scan, we never really use other guideposts (but this may change).
- List<KeyRange> splits = getAllSplits(conn, "MULTI_CF", "e.cpu_utilization IS NOT NULL");
+ List<KeyRange> splits = getAllSplits(conn, "MULTI_CF", "e.cpu_utilization IS NOT NULL", "COUNT(*)");
// Since the E column family is not populated, it won't have as many splits
assertEquals(3, splits.size());
// Same as above for G column family.
- splits = getAllSplits(conn, "MULTI_CF", "g.response_time IS NOT NULL");
+ splits = getAllSplits(conn, "MULTI_CF", "g.response_time IS NOT NULL", "COUNT(*)");
assertEquals(3, splits.size());
} finally {
conn.close();
}
}
-
+
+ @Test
+ public void testGuidePostsRetrievedForMultiCF() throws Exception {
+ Connection conn;
+ PreparedStatement stmt;
+ ResultSet rs;
+
+ long ts = nextTimestamp();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement()
+ .execute(
+ "CREATE TABLE T ( k INTEGER PRIMARY KEY, A.V1 VARCHAR, B.V2 VARCHAR, C.V3 VARCHAR)");
+ conn.close();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30));
+ conn = DriverManager.getConnection(getUrl(), props);
+ stmt = conn.prepareStatement("UPSERT INTO T VALUES(?,?,?,?)");
+ stmt.setInt(1, 1);
+ stmt.setString(2, "A");
+ stmt.setString(3, "B");
+ stmt.setString(4, "C");
+ stmt.execute();
+ conn.commit();
+
+ stmt = conn.prepareStatement("UPSERT INTO T VALUES(?,?,?,?)");
+ stmt.setInt(1, 2);
+ stmt.setString(2, "D");
+ stmt.setString(3, "E");
+ stmt.setString(4, "F");
+ stmt.execute();
+ conn.commit();
+
+ stmt = conn.prepareStatement("UPSERT INTO T(k, A.V1, C.V3) VALUES(?,?,?)");
+ stmt.setInt(1, 3);
+ stmt.setString(2, "E");
+ stmt.setString(3, "X");
+ stmt.execute();
+ conn.commit();
+
+ stmt = conn.prepareStatement("UPSERT INTO T(k, A.V1, C.V3) VALUES(?,?,?)");
+ stmt.setInt(1, 4);
+ stmt.setString(2, "F");
+ stmt.setString(3, "F");
+ stmt.execute();
+ conn.commit();
+
+ conn.close();
+
+ analyzeTable(getUrl(), ts + 50, "T");
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 60));
+ conn = DriverManager.getConnection(getUrl(), props);
+ rs = conn.createStatement().executeQuery("SELECT B.V2 FROM T WHERE B.V2 = 'B'");
+ assertTrue(rs.next());
+ assertEquals("B",rs.getString(1));
+ List<KeyRange> splits = getAllSplits(conn, "T", "C.V3 = 'X'", "A.V1");
+ assertEquals(5, splits.size());
+ splits = getAllSplits(conn, "T", "B.V2 = 'B'", "B.V2");
+ assertEquals(3, splits.size());
+ conn.close();
+ }
+
@Test
public void testCFToDisambiguate2() throws Exception {
long ts = nextTimestamp();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b35cb98c/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
index 7cc2e66..3cf3934 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
@@ -79,7 +79,6 @@ import org.apache.phoenix.schema.PDatum;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.IndexType;
-import org.apache.phoenix.schema.PTable.ViewType;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.RowKeySchema;
@@ -107,6 +106,7 @@ import com.google.common.collect.Sets;
* @since 0.1
*/
public class ProjectionCompiler {
+ private static final Expression NULL_EXPRESSION = LiteralExpression.newConstant(null);
private ProjectionCompiler() {
}
@@ -116,7 +116,9 @@ public class ProjectionCompiler {
}
public static RowProjector compile(StatementContext context, SelectStatement statement, GroupBy groupBy) throws SQLException {
- return compile(context, statement, groupBy, Collections.<PColumn>emptyList());
+ return compile(context, statement, groupBy, Collections.<PColumn>emptyList(),
+ NULL_EXPRESSION// Pass null expression because we don't want empty key value to be projected
+ );
}
private static int getMinPKOffset(PTable table, PName tenantId) {
@@ -338,7 +340,7 @@ public class ProjectionCompiler {
* @return projector used to access row values during scan
* @throws SQLException
*/
- public static RowProjector compile(StatementContext context, SelectStatement statement, GroupBy groupBy, List<? extends PDatum> targetColumns) throws SQLException {
+ public static RowProjector compile(StatementContext context, SelectStatement statement, GroupBy groupBy, List<? extends PDatum> targetColumns, Expression where) throws SQLException {
List<KeyValueColumnExpression> arrayKVRefs = new ArrayList<KeyValueColumnExpression>();
List<ProjectedColumnExpression> arrayProjectedColumnRefs = new ArrayList<ProjectedColumnExpression>();
List<Expression> arrayKVFuncs = new ArrayList<Expression>();
@@ -384,7 +386,7 @@ public class ProjectionCompiler {
} else {
projectAllTableColumns(context, tRef, true, projectedExpressions, projectedColumns, targetColumns);
}
- } else if (node instanceof FamilyWildcardParseNode){
+ } else if (node instanceof FamilyWildcardParseNode) {
if (tableRef == TableRef.EMPTY_TABLE_REF) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_TABLE_SPECIFIED_FOR_WILDCARD_SELECT).build().buildException();
}
@@ -483,16 +485,16 @@ public class ProjectionCompiler {
}
selectVisitor.compile();
- boolean isProjectEmptyKeyValue = (table.getType() != PTableType.VIEW || table.getViewType() != ViewType.MAPPED)
- && !isWildcard;
+ boolean isProjectEmptyKeyValue = false;
if (isWildcard) {
projectAllColumnFamilies(table, scan);
} else {
+ isProjectEmptyKeyValue = where == null || LiteralExpression.isTrue(where) || where.requiresFinalEvaluation();
for (byte[] family : projectedFamilies) {
projectColumnFamily(table, scan, family);
}
}
- return new RowProjector(projectedColumns, estimatedByteSize, isProjectEmptyKeyValue, resolver.hasUDFs());
+ return new RowProjector(projectedColumns, estimatedByteSize, isProjectEmptyKeyValue, resolver.hasUDFs(), isWildcard);
}
private static void projectAllColumnFamilies(PTable table, Scan scan) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b35cb98c/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
index ad65c1c..70bb815 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
@@ -542,7 +542,7 @@ public class QueryCompiler {
Set<SubqueryParseNode> subqueries = Sets.<SubqueryParseNode> newHashSet();
Expression where = WhereCompiler.compile(context, select, viewWhere, subqueries);
context.setResolver(resolver); // recover resolver
- RowProjector projector = ProjectionCompiler.compile(context, select, groupBy, asSubquery ? Collections.<PDatum>emptyList() : targetColumns);
+ RowProjector projector = ProjectionCompiler.compile(context, select, groupBy, asSubquery ? Collections.<PDatum>emptyList() : targetColumns, where);
OrderBy orderBy = OrderByCompiler.compile(context, select, groupBy, limit, projector, groupBy == GroupBy.EMPTY_GROUP_BY ? innerPlanTupleProjector : null, isInRowKeyOrder);
// Final step is to build the query plan
if (!asSubquery) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b35cb98c/phoenix-core/src/main/java/org/apache/phoenix/compile/RowProjector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/RowProjector.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/RowProjector.java
index 99ab5d4..2123788 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/RowProjector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/RowProjector.java
@@ -25,7 +25,6 @@ import java.util.List;
import org.apache.phoenix.expression.Determinism;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.expression.visitor.CloneNonDeterministicExpressionVisitor;
-import org.apache.phoenix.schema.AmbiguousColumnException;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.util.SchemaUtil;
@@ -50,12 +49,13 @@ public class RowProjector {
private final boolean allCaseSensitive;
private final boolean someCaseSensitive;
private final int estimatedSize;
+ private final boolean isProjectAll;
private final boolean isProjectEmptyKeyValue;
private final boolean cloneRequired;
private final boolean hasUDFs;
public RowProjector(RowProjector projector, boolean isProjectEmptyKeyValue) {
- this(projector.getColumnProjectors(), projector.getEstimatedRowByteSize(), isProjectEmptyKeyValue, projector.hasUDFs);
+ this(projector.getColumnProjectors(), projector.getEstimatedRowByteSize(), isProjectEmptyKeyValue, projector.hasUDFs, projector.isProjectAll);
}
/**
* Construct RowProjector based on a list of ColumnProjectors.
@@ -65,7 +65,7 @@ public class RowProjector {
* @param estimatedRowSize
*/
public RowProjector(List<? extends ColumnProjector> columnProjectors, int estimatedRowSize, boolean isProjectEmptyKeyValue) {
- this(columnProjectors, estimatedRowSize, isProjectEmptyKeyValue, false);
+ this(columnProjectors, estimatedRowSize, isProjectEmptyKeyValue, false, false);
}
/**
* Construct RowProjector based on a list of ColumnProjectors.
@@ -76,7 +76,7 @@ public class RowProjector {
* @param isProjectEmptyKeyValue
* @param hasUDFs
*/
- public RowProjector(List<? extends ColumnProjector> columnProjectors, int estimatedRowSize, boolean isProjectEmptyKeyValue, boolean hasUDFs) {
+ public RowProjector(List<? extends ColumnProjector> columnProjectors, int estimatedRowSize, boolean isProjectEmptyKeyValue, boolean hasUDFs, boolean isProjectAll) {
this.columnProjectors = Collections.unmodifiableList(columnProjectors);
int position = columnProjectors.size();
reverseIndex = ArrayListMultimap.<String, Integer>create();
@@ -95,6 +95,7 @@ public class RowProjector {
this.someCaseSensitive = someCaseSensitive;
this.estimatedSize = estimatedRowSize;
this.isProjectEmptyKeyValue = isProjectEmptyKeyValue;
+ this.isProjectAll = isProjectAll;
this.hasUDFs = hasUDFs;
boolean hasPerInvocationExpression = false;
if (!hasUDFs) {
@@ -129,14 +130,17 @@ public class RowProjector {
}
}
return new RowProjector(clonedColProjectors,
- this.getEstimatedRowByteSize(),
- this.isProjectEmptyKeyValue(), this.hasUDFs);
+ this.estimatedSize, this.isProjectEmptyKeyValue, this.hasUDFs, this.isProjectAll);
}
- public boolean isProjectEmptyKeyValue() {
+ public boolean projectEveryRow() {
return isProjectEmptyKeyValue;
}
+ public boolean projectEverything() {
+ return isProjectAll;
+ }
+
public List<? extends ColumnProjector> getColumnProjectors() {
return columnProjectors;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b35cb98c/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
index 80c4b89..9f4562a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
@@ -280,7 +280,7 @@ public class StatementContext {
whereConditionColumns.add(new Pair<byte[], byte[]>(cf, q));
}
- public List<Pair<byte[], byte[]>> getWhereCoditionColumns() {
+ public List<Pair<byte[], byte[]>> getWhereConditionColumns() {
return whereConditionColumns;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b35cb98c/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 19c60bf..21f082f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -25,6 +25,7 @@ import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -158,32 +159,38 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
} else {
FilterableStatement statement = plan.getStatement();
RowProjector projector = plan.getProjector();
- boolean keyOnlyFilter = familyMap.isEmpty() && context.getWhereCoditionColumns().isEmpty();
- if (projector.isProjectEmptyKeyValue()) {
+ boolean keyOnlyFilter = familyMap.isEmpty() && context.getWhereConditionColumns().isEmpty();
+ if (!projector.projectEverything()) {
// If nothing projected into scan and we only have one column family, just allow everything
// to be projected and use a FirstKeyOnlyFilter to skip from row to row. This turns out to
// be quite a bit faster.
// Where condition columns also will get added into familyMap
// When where conditions are present, we can not add FirstKeyOnlyFilter at beginning.
- if (familyMap.isEmpty() && context.getWhereCoditionColumns().isEmpty()
+ if (familyMap.isEmpty() && context.getWhereConditionColumns().isEmpty()
&& table.getColumnFamilies().size() == 1) {
// Project the one column family. We must project a column family since it's possible
// that there are other non declared column families that we need to ignore.
scan.addFamily(table.getColumnFamilies().get(0).getName().getBytes());
} else {
- byte[] ecf = SchemaUtil.getEmptyColumnFamily(table);
- // Project empty key value unless the column family containing it has
- // been projected in its entirety.
- if (!familyMap.containsKey(ecf) || familyMap.get(ecf) != null) {
- scan.addColumn(ecf, QueryConstants.EMPTY_COLUMN_BYTES);
+ if (projector.projectEveryRow()) {
+ byte[] ecf = SchemaUtil.getEmptyColumnFamily(table);
+ // Project empty key value unless the column family containing it has
+ // been projected in its entirety.
+ if (!familyMap.containsKey(ecf) || familyMap.get(ecf) != null) {
+ scan.addColumn(ecf, QueryConstants.EMPTY_COLUMN_BYTES);
+ }
}
}
- } else if (table.getViewType() == ViewType.MAPPED) {
- // Since we don't have the empty key value in MAPPED tables, we must select all CFs in HRS. But only the
- // selected column values are returned back to client
- for (PColumnFamily family : table.getColumnFamilies()) {
- scan.addFamily(family.getName().getBytes());
- }
+ if (table.getViewType() == ViewType.MAPPED) {
+ if (projector.projectEveryRow()) {
+ // Since we don't have the empty key value in MAPPED tables,
+ // we must select all CFs in HRS. However, only the
+ // selected column values are returned back to client.
+ for (PColumnFamily family : table.getColumnFamilies()) {
+ scan.addFamily(family.getName().getBytes());
+ }
+ }
+ }
}
// Add FirstKeyOnlyFilter if there are no references to key value columns
if (keyOnlyFilter) {
@@ -236,7 +243,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
new TreeMap<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>>();
Set<byte[]> conditionOnlyCfs = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
int referencedCfCount = familyMap.size();
- for (Pair<byte[], byte[]> whereCol : context.getWhereCoditionColumns()) {
+ for (Pair<byte[], byte[]> whereCol : context.getWhereConditionColumns()) {
if (!(familyMap.containsKey(whereCol.getFirst()))) {
referencedCfCount++;
}
@@ -267,7 +274,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
}
}
// Making sure that where condition CFs are getting scanned at HRS.
- for (Pair<byte[], byte[]> whereCol : context.getWhereCoditionColumns()) {
+ for (Pair<byte[], byte[]> whereCol : context.getWhereConditionColumns()) {
if (useOptimization) {
if (!(familyMap.containsKey(whereCol.getFirst()))) {
scan.addFamily(whereCol.getFirst());
@@ -350,7 +357,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
return guideIndex;
}
- private List<byte[]> getGuidePosts() {
+ private List<byte[]> getGuidePosts(Set<byte[]> whereConditions) {
/*
* Don't use guide posts if:
* 1) We're doing a point lookup, as HBase is fast enough at those
@@ -368,22 +375,26 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
byte[] defaultCF = SchemaUtil.getEmptyColumnFamily(getTable());
if (table.getColumnFamilies().isEmpty()) {
// For sure we can get the defaultCF from the table
- if (guidePostMap.get(defaultCF) != null) {
- gps = guidePostMap.get(defaultCF).getGuidePosts();
- }
+ gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF);
} else {
- Scan scan = context.getScan();
- if (scan.getFamilyMap().size() > 0 && !scan.getFamilyMap().containsKey(defaultCF)) {
- // If default CF is not used in scan, use first CF referenced in scan
- GuidePostsInfo guidePostsInfo = guidePostMap.get(scan.getFamilyMap().keySet().iterator().next());
- if (guidePostsInfo != null) {
- gps = guidePostsInfo.getGuidePosts();
+ byte[] familyInWhere = null;
+ if (!whereConditions.isEmpty()) {
+ if (whereConditions.contains(defaultCF)) {
+ gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF);
+ } else {
+ familyInWhere = whereConditions.iterator().next();
+ if(familyInWhere != null) {
+ GuidePostsInfo guidePostsInfo = guidePostMap.get(familyInWhere);
+ if (guidePostsInfo != null) {
+ gps = guidePostsInfo.getGuidePosts();
+ } else {
+ // As there are no guideposts collected for the where family we go with the default CF
+ gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF);
+ }
}
+ }
} else {
- // Otherwise, favor use of default CF.
- if (guidePostMap.get(defaultCF) != null) {
- gps = guidePostMap.get(defaultCF).getGuidePosts();
- }
+ gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF);
}
}
if (gps == null) {
@@ -391,6 +402,13 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
}
return gps;
}
+
+ private List<byte[]> getDefaultFamilyGuidePosts(Map<byte[], GuidePostsInfo> guidePostMap, byte[] defaultCF) {
+ if (guidePostMap.get(defaultCF) != null) {
+ return guidePostMap.get(defaultCF).getGuidePosts();
+ }
+ return null;
+ }
private static String toString(List<byte[]> gps) {
StringBuilder buf = new StringBuilder(gps.size() * 100);
@@ -439,7 +457,11 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
PTable table = getTable();
boolean isSalted = table.getBucketNum() != null;
boolean isLocalIndex = table.getIndexType() == IndexType.LOCAL;
- List<byte[]> gps = getGuidePosts();
+ HashSet<byte[]> whereConditions = new HashSet<byte[]>(context.getWhereConditionColumns().size());
+ for(Pair<byte[], byte[]> where : context.getWhereConditionColumns()) {
+ whereConditions.add(where.getFirst());
+ }
+ List<byte[]> gps = getGuidePosts(whereConditions);
if (logger.isDebugEnabled()) {
logger.debug("Guideposts: " + toString(gps));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b35cb98c/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
index bea222b..9411549 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
@@ -441,6 +441,12 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest {
return plan.getContext().getScan();
}
+ private Scan projectQuery(String query) throws SQLException {
+ QueryPlan plan = getQueryPlan(query, Collections.emptyList());
+ plan.iterator(); // Forces projection
+ return plan.getContext().getScan();
+ }
+
private QueryPlan getQueryPlan(String query, List<Object> binds) throws SQLException {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props);
@@ -449,7 +455,8 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest {
for (Object bind : binds) {
statement.setObject(1, bind);
}
- return statement.compileQuery(query);
+ QueryPlan plan = statement.compileQuery(query);
+ return plan;
} finally {
conn.close();
}
@@ -2146,5 +2153,29 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest {
}
}
+ private static void assertFamilies(Scan s, String... families) {
+ assertEquals(families.length, s.getFamilyMap().size());
+ for (String fam : families) {
+ byte[] cf = Bytes.toBytes(fam);
+ assertTrue("Expected to contain " + fam, s.getFamilyMap().containsKey(cf));
+ }
+ }
+
+ @Test
+ public void testProjection() throws SQLException {
+ Connection conn = DriverManager.getConnection(getUrl());
+ try {
+ conn.createStatement().execute("CREATE TABLE t(k INTEGER PRIMARY KEY, a.v1 VARCHAR, b.v2 VARCHAR, c.v3 VARCHAR)");
+ assertFamilies(projectQuery("SELECT k FROM t"), "A");
+ assertFamilies(projectQuery("SELECT k FROM t WHERE k = 5"), "A");
+ assertFamilies(projectQuery("SELECT v2 FROM t WHERE k = 5"), "A", "B");
+ assertFamilies(projectQuery("SELECT v2 FROM t WHERE v2 = 'a'"), "B");
+ assertFamilies(projectQuery("SELECT v3 FROM t WHERE v2 = 'a'"), "B", "C");
+ assertFamilies(projectQuery("SELECT v3 FROM t WHERE v2 = 'a' AND v3 is null"), "A", "B", "C");
+ } finally {
+ conn.close();
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b35cb98c/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index 451ce62..e66f8ca 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -490,20 +490,20 @@ public class TestUtil {
}
public static List<KeyRange> getAllSplits(Connection conn, String tableName) throws SQLException {
- return getSplits(conn, tableName, null, null, null, null);
+ return getSplits(conn, tableName, null, null, null, null, null);
}
- public static List<KeyRange> getAllSplits(Connection conn, String tableName, String where) throws SQLException {
- return getSplits(conn, tableName, null, null, null, where);
+ public static List<KeyRange> getAllSplits(Connection conn, String tableName, String where, String selectClause) throws SQLException {
+ return getSplits(conn, tableName, null, null, null, where, selectClause);
}
- public static List<KeyRange> getSplits(Connection conn, String tableName, String pkCol, byte[] lowerRange, byte[] upperRange, String whereClauseSuffix) throws SQLException {
+ public static List<KeyRange> getSplits(Connection conn, String tableName, String pkCol, byte[] lowerRange, byte[] upperRange, String whereClauseSuffix, String selectClause) throws SQLException {
String whereClauseStart =
(lowerRange == null && upperRange == null ? "" :
" WHERE " + ((lowerRange != null ? (pkCol + " >= ? " + (upperRange != null ? " AND " : "")) : "")
+ (upperRange != null ? (pkCol + " < ?") : "" )));
String whereClause = whereClauseSuffix == null ? whereClauseStart : whereClauseStart.length() == 0 ? (" WHERE " + whereClauseSuffix) : (" AND " + whereClauseSuffix);
- String query = "SELECT /*+ NO_INDEX */ COUNT(*) FROM " + tableName + whereClause;
+ String query = "SELECT /*+ NO_INDEX */ "+selectClause+" FROM " + tableName + whereClause;
PhoenixPreparedStatement pstmt = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class);
if (lowerRange != null) {
pstmt.setBytes(1, lowerRange);
@@ -548,7 +548,7 @@ public class TestUtil {
}
public static List<KeyRange> getSplits(Connection conn, byte[] lowerRange, byte[] upperRange) throws SQLException {
- return getSplits(conn, STABLE_NAME, STABLE_PK_NAME, lowerRange, upperRange, null);
+ return getSplits(conn, STABLE_NAME, STABLE_PK_NAME, lowerRange, upperRange, null, "COUNT(*)");
}
public static List<KeyRange> getAllSplits(Connection conn) throws SQLException {
[39/50] [abbrv] phoenix git commit: PHOENIX-2558 Fix server-side
cache memory leaks
Posted by sa...@apache.org.
PHOENIX-2558 Fix server-side cache memory leaks
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/1cd1d268
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/1cd1d268
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/1cd1d268
Branch: refs/heads/4.x-HBase-1.0
Commit: 1cd1d26824641a6125236a5d02f43fc22c6df5c3
Parents: 31a414c
Author: James Taylor <jt...@salesforce.com>
Authored: Sun Jan 3 17:08:31 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Jan 4 09:36:39 2016 -0800
----------------------------------------------------------------------
.../end2end/BaseTenantSpecificViewIndexIT.java | 2 --
.../org/apache/phoenix/end2end/HashJoinIT.java | 14 +++++++++++++
.../apache/phoenix/end2end/HashJoinMoreIT.java | 15 +++++++++++++
.../end2end/QueryDatabaseMetaDataIT.java | 18 +++++++---------
.../apache/phoenix/end2end/QueryTimeoutIT.java | 14 +++++++++++++
.../apache/phoenix/cache/ServerCacheClient.java | 13 +++++++++++-
.../apache/phoenix/execute/MutationState.java | 16 +++++++-------
.../phoenix/iterate/BaseResultIterators.java | 22 +++++++++++++++++++-
.../MaterializedComparableResultIterator.java | 3 +++
.../apache/phoenix/jdbc/PhoenixResultSet.java | 3 +++
10 files changed, 98 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1cd1d268/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java
index b450643..c10afa6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java
@@ -19,7 +19,6 @@ package org.apache.phoenix.end2end;
import static com.google.common.collect.Sets.newHashSet;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -178,6 +177,5 @@ public class BaseTenantSpecificViewIndexIT extends BaseHBaseManagedTimeIT {
Arrays.<Object>asList(1,7, valuePrefix + "v2-1"),
Arrays.<Object>asList(1,9, valuePrefix + "v2-1"));
assertValuesEqualsResultSet(rs,expectedResultsA);
- assertFalse(rs.next());
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1cd1d268/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
index 3d8b006..dd7f6ba 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
@@ -50,12 +50,14 @@ import java.util.Map;
import java.util.Properties;
import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.TableAlreadyExistsException;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -78,6 +80,18 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
this.plans = plans;
}
+ @After
+ public void assertNoUnfreedMemory() throws SQLException {
+ Connection conn = DriverManager.getConnection(getUrl());
+ try {
+ long unfreedBytes = conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache();
+ assertEquals(0,unfreedBytes);
+ } finally {
+ conn.close();
+ }
+ }
+
+
@BeforeClass
@Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
public static void doSetup() throws Exception {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1cd1d268/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinMoreIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinMoreIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinMoreIT.java
index a8bb977..98264f0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinMoreIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinMoreIT.java
@@ -27,14 +27,17 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
+import java.sql.SQLException;
import java.sql.Statement;
import java.util.Map;
import java.util.Properties;
+import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -101,6 +104,17 @@ public class HashJoinMoreIT extends BaseHBaseManagedTimeIT {
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
+ @After
+ public void assertNoUnfreedMemory() throws SQLException {
+ Connection conn = DriverManager.getConnection(getUrl());
+ try {
+ long unfreedBytes = conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache();
+ assertEquals(0,unfreedBytes);
+ } finally {
+ conn.close();
+ }
+ }
+
@Test
public void testJoinOverSaltedTables() throws Exception {
String tempTableNoSalting = "TEMP_TABLE_NO_SALTING";
@@ -555,6 +569,7 @@ public class HashJoinMoreIT extends BaseHBaseManagedTimeIT {
assertTrue(rs.next());
assertEquals(rs.getInt(1), 5);
assertFalse(rs.next());
+ rs.close();
rs = conn.createStatement().executeQuery(
"SELECT * FROM INVENTORY RIGHT JOIN PRODUCT_IDS ON (PRODUCT_ID = INVENTORY.ID)");
assertTrue(rs.next());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1cd1d268/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
index 2fdccf6..ba83e6a 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
@@ -48,13 +48,11 @@ import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.Arrays;
import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
@@ -69,15 +67,15 @@ import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryServices;
-import org.apache.phoenix.schema.types.PChar;
import org.apache.phoenix.schema.ColumnNotFoundException;
-import org.apache.phoenix.schema.types.PDecimal;
-import org.apache.phoenix.schema.types.PInteger;
-import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.schema.PTable.ViewType;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.ReadOnlyTableException;
import org.apache.phoenix.schema.TableNotFoundException;
+import org.apache.phoenix.schema.types.PChar;
+import org.apache.phoenix.schema.types.PDecimal;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
@@ -151,17 +149,15 @@ public class QueryDatabaseMetaDataIT extends BaseClientManagedTimeIT {
rs = dbmd.getTables(null, CUSTOM_ENTITY_DATA_SCHEMA_NAME, CUSTOM_ENTITY_DATA_NAME, null);
assertTrue(rs.next());
- assertEquals(rs.getString("TABLE_SCHEM"),CUSTOM_ENTITY_DATA_SCHEMA_NAME);
- assertEquals(rs.getString("TABLE_NAME"),CUSTOM_ENTITY_DATA_NAME);
- assertEquals(PTableType.TABLE.toString(), rs.getString("TABLE_TYPE"));
- assertFalse(rs.next());
-
try {
rs.getString("RANDOM_COLUMN_NAME");
fail();
} catch (ColumnNotFoundException e) {
// expected
}
+ assertEquals(rs.getString("TABLE_SCHEM"),CUSTOM_ENTITY_DATA_SCHEMA_NAME);
+ assertEquals(rs.getString("TABLE_NAME"),CUSTOM_ENTITY_DATA_NAME);
+ assertEquals(PTableType.TABLE.toString(), rs.getString("TABLE_TYPE"));
assertFalse(rs.next());
rs = dbmd.getTables(null, "", "_TABLE", new String[] {PTableType.TABLE.toString()});
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1cd1d268/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryTimeoutIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryTimeoutIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryTimeoutIT.java
index ba7b461..ccd6530 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryTimeoutIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryTimeoutIT.java
@@ -28,15 +28,18 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
+import java.sql.SQLException;
import java.sql.SQLTimeoutException;
import java.util.Map;
import java.util.Properties;
import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -56,6 +59,17 @@ public class QueryTimeoutIT extends BaseOwnClusterHBaseManagedTimeIT {
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
+ @After
+ public void assertNoUnfreedMemory() throws SQLException {
+ Connection conn = DriverManager.getConnection(getUrl());
+ try {
+ long unfreedBytes = conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache();
+ assertEquals(0,unfreedBytes);
+ } finally {
+ conn.close();
+ }
+ }
+
@Test
public void testQueryTimeout() throws Exception {
int nRows = 30000;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1cd1d268/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java b/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
index f188ab2..424482a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
@@ -301,6 +301,7 @@ public class ServerCacheClient {
ConnectionQueryServices services = connection.getQueryServices();
Throwable lastThrowable = null;
TableRef cacheUsingTableRef = cacheUsingTableRefMap.get(Bytes.mapKey(cacheId));
+ final PTable cacheUsingTable = cacheUsingTableRef.getTable();
byte[] tableName = cacheUsingTableRef.getTable().getPhysicalName().getBytes();
HTableInterface iterateOverTable = services.getTable(tableName);
try {
@@ -326,7 +327,17 @@ public class ServerCacheClient {
new BlockingRpcCallback<RemoveServerCacheResponse>();
RemoveServerCacheRequest.Builder builder = RemoveServerCacheRequest.newBuilder();
if(connection.getTenantId() != null){
- builder.setTenantId(ByteStringer.wrap(connection.getTenantId().getBytes()));
+ try {
+ byte[] tenantIdBytes =
+ ScanUtil.getTenantIdBytes(
+ cacheUsingTable.getRowKeySchema(),
+ cacheUsingTable.getBucketNum()!=null,
+ connection.getTenantId(),
+ cacheUsingTable.isMultiTenant());
+ builder.setTenantId(ByteStringer.wrap(tenantIdBytes));
+ } catch (SQLException e) {
+ new IOException(e);
+ }
}
builder.setCacheId(ByteStringer.wrap(cacheId));
instance.removeServerCache(controller, builder.build(), rpcCallback);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1cd1d268/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
index 8ae9481..bdc9460 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -81,6 +81,7 @@ import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.LogUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.SQLCloseable;
+import org.apache.phoenix.util.SQLCloseables;
import org.apache.phoenix.util.ServerUtil;
import org.apache.phoenix.util.TransactionUtil;
import org.slf4j.Logger;
@@ -703,6 +704,8 @@ public class MutationState implements SQLCloseable {
*/
@Override
public void delete(List<Delete> deletes) throws IOException {
+ ServerCache cache = null;
+ SQLException sqlE = null;
try {
PTable table = tableRef.getTable();
List<PTable> indexes = table.getIndexes();
@@ -736,12 +739,16 @@ public class MutationState implements SQLCloseable {
IndexMaintainer.serializeAdditional(table, indexMetaDataPtr, keyValueIndexes, connection);
}
if (attachMetaData) {
- setMetaDataOnMutations(tableRef, deletes, indexMetaDataPtr);
+ cache = setMetaDataOnMutations(tableRef, deletes, indexMetaDataPtr);
}
}
delegate.delete(deletes);
} catch (SQLException e) {
throw new IOException(e);
+ } finally {
+ if (cache != null) {
+ SQLCloseables.closeAllQuietly(Collections.singletonList(cache));
+ }
}
}
}
@@ -793,10 +800,7 @@ public class MutationState implements SQLCloseable {
int retryCount = 0;
boolean shouldRetry = false;
do {
- ServerCache cache = null;
- if (isDataTable) {
- cache = setMetaDataOnMutations(tableRef, mutationList, indexMetaDataPtr);
- }
+ final ServerCache cache = isDataTable ? setMetaDataOnMutations(tableRef, mutationList, indexMetaDataPtr) : null;
// If we haven't retried yet, retry for this case only, as it's possible that
// a split will occur after we send the index metadata cache to all known
@@ -879,8 +883,6 @@ public class MutationState implements SQLCloseable {
}
}
if (sqlE != null) {
- // clear pending mutations
- mutations.clear();
throw sqlE;
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1cd1d268/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 21f082f..2806acd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -704,6 +704,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
// to get into a funk. Instead, just cancel queued work.
boolean cancelledWork = false;
try {
+ List<Future<PeekingResultIterator>> futuresToClose = Lists.newArrayListWithExpectedSize(getSplits().size());
for (List<List<Pair<Scan,Future<PeekingResultIterator>>>> futures : allFutures) {
for (List<Pair<Scan,Future<PeekingResultIterator>>> futureScans : futures) {
for (Pair<Scan,Future<PeekingResultIterator>> futurePair : futureScans) {
@@ -712,16 +713,35 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
if (futurePair != null) {
Future<PeekingResultIterator> future = futurePair.getSecond();
if (future != null) {
- future.cancel(false);
+ if (future.cancel(false)) {
+ cancelledWork = true;
+ } else {
+ futuresToClose.add(future);
+ }
}
}
}
}
}
+ // Wait for already started tasks to complete as we can't interrupt them without
+ // leaving our HConnection in a funky state.
+ for (Future<PeekingResultIterator> future : futuresToClose) {
+ try {
+ PeekingResultIterator iterator = future.get();
+ iterator.close();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ } catch (ExecutionException e) {
+ logger.info("Failed to execute task during cancel", e);
+ continue;
+ }
+ }
} finally {
if (cancelledWork) {
context.getConnection().getQueryServices().getExecutor().purge();
}
+ allFutures.clear();
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1cd1d268/phoenix-core/src/main/java/org/apache/phoenix/iterate/MaterializedComparableResultIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/MaterializedComparableResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/MaterializedComparableResultIterator.java
index 093a098..a76f1e3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/MaterializedComparableResultIterator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/MaterializedComparableResultIterator.java
@@ -45,16 +45,19 @@ public class MaterializedComparableResultIterator
this.current = delegate.peek();
}
+ @Override
public Tuple next() throws SQLException {
Tuple next = delegate.next();
this.current = delegate.peek();
return next;
}
+ @Override
public Tuple peek() throws SQLException {
return delegate.peek();
}
+ @Override
public void close() throws SQLException {
delegate.close();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1cd1d268/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java
index a3ce1a1..47c17ae 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java
@@ -776,6 +776,9 @@ public class PhoenixResultSet implements ResultSet, SQLCloseable {
overAllQueryMetrics.startResultSetWatch();
}
currentRow = scanner.next();
+ if (currentRow == null) {
+ close();
+ }
rowProjector.reset();
} catch (RuntimeException e) {
// FIXME: Expression.evaluate does not throw SQLException
[49/50] [abbrv] phoenix git commit: PHOENIX-2357 Prevent queries from
failing because of scanners running into lease expiration
Posted by sa...@apache.org.
PHOENIX-2357 Prevent queries from failing because of scanners running into lease expiration
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/5fb51ca5
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/5fb51ca5
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/5fb51ca5
Branch: refs/heads/4.x-HBase-1.0
Commit: 5fb51ca5ebd97493e299ffc1e3d7c35fb68be612
Parents: 7f555aa
Author: Samarth <sa...@salesforce.com>
Authored: Tue Jan 5 23:05:06 2016 -0800
Committer: Samarth <sa...@salesforce.com>
Committed: Tue Jan 5 23:05:06 2016 -0800
----------------------------------------------------------------------
.../DelayedTableResultIteratorFactory.java | 71 +++++++
.../iterate/MockParallelIteratorFactory.java | 2 +-
.../phoenix/iterate/MockResultIterator.java | 66 +++++++
.../iterate/MockTableResultIterator.java | 66 -------
.../iterate/RenewLeaseOnlyTableIterator.java | 63 ++++++
.../phoenix/iterate/ScannerLeaseRenewalIT.java | 176 +++++++++++++++++
.../phoenix/iterate/BaseResultIterators.java | 2 +-
.../phoenix/iterate/ChunkedResultIterator.java | 3 +-
.../DefaultTableResultIteratorFactory.java | 35 ++++
.../phoenix/iterate/ParallelIterators.java | 11 +-
.../phoenix/iterate/ScanningResultIterator.java | 19 +-
.../apache/phoenix/iterate/SerialIterators.java | 7 +-
.../phoenix/iterate/TableResultIterator.java | 147 +++++++++-----
.../iterate/TableResultIteratorFactory.java | 29 +++
.../apache/phoenix/jdbc/PhoenixConnection.java | 40 +++-
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 3 +-
.../phoenix/mapreduce/PhoenixRecordReader.java | 3 +-
.../phoenix/query/ConnectionQueryServices.java | 8 +-
.../query/ConnectionQueryServicesImpl.java | 196 +++++++++++++++++--
.../query/ConnectionlessQueryServicesImpl.java | 10 +
.../query/DelegateConnectionQueryServices.java | 11 ++
.../org/apache/phoenix/query/QueryServices.java | 5 +
.../phoenix/query/QueryServicesOptions.java | 25 ++-
.../java/org/apache/phoenix/query/BaseTest.java | 2 +-
.../phoenix/query/QueryServicesTestImpl.java | 4 +-
.../phoenix/query/ScannerLeaseRenewalTest.java | 132 +++++++++++++
.../apache/phoenix/util/MetaDataUtilTest.java | 1 +
pom.xml | 2 +-
28 files changed, 974 insertions(+), 165 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/it/java/org/apache/phoenix/iterate/DelayedTableResultIteratorFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/iterate/DelayedTableResultIteratorFactory.java b/phoenix-core/src/it/java/org/apache/phoenix/iterate/DelayedTableResultIteratorFactory.java
new file mode 100644
index 0000000..6545a45
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/iterate/DelayedTableResultIteratorFactory.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.iterate;
+
+import java.sql.SQLException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.phoenix.compile.StatementContext;
+import org.apache.phoenix.monitoring.CombinableMetric;
+import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.schema.tuple.Tuple;
+
+public class DelayedTableResultIteratorFactory implements TableResultIteratorFactory {
+
+ private final long delay;
+
+ public DelayedTableResultIteratorFactory(long delay) {
+ this.delay = delay;
+ }
+
+ @Override
+ public TableResultIterator newIterator(StatementContext context, TableRef tableRef, Scan scan,
+ CombinableMetric scanMetrics, long renewLeaseThreshold) throws SQLException {
+ return new DelayedTableResultIterator(context, tableRef, scan, scanMetrics, renewLeaseThreshold);
+ }
+
+ private class DelayedTableResultIterator extends TableResultIterator {
+ public DelayedTableResultIterator (StatementContext context, TableRef tableRef, Scan scan, CombinableMetric scanMetrics, long renewLeaseThreshold) throws SQLException {
+ super(context.getConnection().getMutationState(), tableRef, scan, scanMetrics, renewLeaseThreshold);
+ }
+
+ @Override
+ public synchronized void initScanner() throws SQLException {
+ super.initScanner();
+ }
+
+ @Override
+ public Tuple next() throws SQLException {
+ delay();
+ return super.next();
+ }
+
+ private void delay() {
+ try {
+ new CountDownLatch(1).await(delay, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/it/java/org/apache/phoenix/iterate/MockParallelIteratorFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/iterate/MockParallelIteratorFactory.java b/phoenix-core/src/it/java/org/apache/phoenix/iterate/MockParallelIteratorFactory.java
index 1161bae..d8a08e6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/iterate/MockParallelIteratorFactory.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/iterate/MockParallelIteratorFactory.java
@@ -34,7 +34,7 @@ public class MockParallelIteratorFactory implements ParallelIteratorFactory {
@Override
public PeekingResultIterator newIterator(StatementContext context, ResultIterator scanner, Scan scan,
String physicalTableName) throws SQLException {
- return new MockTableResultIterator(String.valueOf(counter.incrementAndGet()), table);
+ return new MockResultIterator(String.valueOf(counter.incrementAndGet()), table);
}
public void setTable(PTable table) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/it/java/org/apache/phoenix/iterate/MockResultIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/iterate/MockResultIterator.java b/phoenix-core/src/it/java/org/apache/phoenix/iterate/MockResultIterator.java
new file mode 100644
index 0000000..b26dfcf
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/iterate/MockResultIterator.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.iterate;
+
+import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN;
+import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.execute.TupleProjector;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.tuple.ResultTuple;
+import org.apache.phoenix.schema.tuple.Tuple;
+
+/**
+ * Mock result iterator that returns its id as a string in a {@code Tuple} when {@link #next()} and {@link #peek()} are called.
+ */
+public class MockResultIterator implements PeekingResultIterator {
+
+ private final Tuple tuple;
+
+ public MockResultIterator(String id, PTable table) {
+ TupleProjector projector = new TupleProjector(table);
+ List<Cell> result = new ArrayList<>();
+ result.add(new KeyValue(Bytes.toBytes(id), SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(id)));
+ this.tuple = projector.projectResults(new ResultTuple(Result.create(result)));
+ }
+
+ @Override
+ public Tuple next() throws SQLException {
+ return tuple;
+ }
+
+ @Override
+ public void explain(List<String> planSteps) {}
+
+ @Override
+ public void close() throws SQLException {}
+
+ @Override
+ public Tuple peek() throws SQLException {
+ return tuple;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/it/java/org/apache/phoenix/iterate/MockTableResultIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/iterate/MockTableResultIterator.java b/phoenix-core/src/it/java/org/apache/phoenix/iterate/MockTableResultIterator.java
deleted file mode 100644
index f0da101..0000000
--- a/phoenix-core/src/it/java/org/apache/phoenix/iterate/MockTableResultIterator.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.iterate;
-
-import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN;
-import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY;
-
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.execute.TupleProjector;
-import org.apache.phoenix.schema.PTable;
-import org.apache.phoenix.schema.tuple.ResultTuple;
-import org.apache.phoenix.schema.tuple.Tuple;
-
-/**
- * Mock result iterator that returns its id as a string in a {@code Tuple} when {@link #next()} and {@link #peek()} are called.
- */
-public class MockTableResultIterator implements PeekingResultIterator {
-
- private final Tuple tuple;
-
- public MockTableResultIterator(String id, PTable table) {
- TupleProjector projector = new TupleProjector(table);
- List<Cell> result = new ArrayList<>();
- result.add(new KeyValue(Bytes.toBytes(id), SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(id)));
- this.tuple = projector.projectResults(new ResultTuple(Result.create(result)));
- }
-
- @Override
- public Tuple next() throws SQLException {
- return tuple;
- }
-
- @Override
- public void explain(List<String> planSteps) {}
-
- @Override
- public void close() throws SQLException {}
-
- @Override
- public Tuple peek() throws SQLException {
- return tuple;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/it/java/org/apache/phoenix/iterate/RenewLeaseOnlyTableIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/iterate/RenewLeaseOnlyTableIterator.java b/phoenix-core/src/it/java/org/apache/phoenix/iterate/RenewLeaseOnlyTableIterator.java
new file mode 100644
index 0000000..5fa4126
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/iterate/RenewLeaseOnlyTableIterator.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.iterate;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.CLOSED;
+import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.NOT_RENEWED;
+import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.RENEWED;
+import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.THRESHOLD_NOT_REACHED;
+
+import java.sql.SQLException;
+
+public class RenewLeaseOnlyTableIterator extends TableResultIterator {
+
+ private final int numberOfLeaseRenewals;
+ private final int thresholdNotReachedAt;
+ private final int doNotRenewLeaseAt;
+ private int counter = 0;
+ private RenewLeaseStatus lastRenewLeaseStatus;
+
+ public RenewLeaseOnlyTableIterator(int renewLeaseCount, int skipRenewLeaseAt, int doNotRenewLeaseAt) throws SQLException {
+ super();
+ checkArgument(renewLeaseCount >= skipRenewLeaseAt);
+ this.numberOfLeaseRenewals = renewLeaseCount;
+ this.thresholdNotReachedAt = skipRenewLeaseAt;
+ this.doNotRenewLeaseAt = doNotRenewLeaseAt;
+ }
+
+ @Override
+ public RenewLeaseStatus renewLease() {
+ counter++;
+ if (counter == thresholdNotReachedAt) {
+ lastRenewLeaseStatus = THRESHOLD_NOT_REACHED;
+ } else if (counter == doNotRenewLeaseAt) {
+ lastRenewLeaseStatus = NOT_RENEWED;
+ } else if (counter <= numberOfLeaseRenewals) {
+ lastRenewLeaseStatus = RENEWED;
+ } else {
+ lastRenewLeaseStatus = CLOSED;
+ }
+ return lastRenewLeaseStatus;
+ }
+
+ public RenewLeaseStatus getLastRenewLeaseStatus() {
+ return lastRenewLeaseStatus;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/it/java/org/apache/phoenix/iterate/ScannerLeaseRenewalIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/iterate/ScannerLeaseRenewalIT.java b/phoenix-core/src/it/java/org/apache/phoenix/iterate/ScannerLeaseRenewalIT.java
new file mode 100644
index 0000000..7b42a77
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/iterate/ScannerLeaseRenewalIT.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.iterate;
+
+import static org.apache.hadoop.hbase.HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD;
+import static org.apache.phoenix.query.QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.RENEW_LEASE_ENABLED;
+import static org.apache.phoenix.query.QueryServices.RENEW_LEASE_THREAD_POOL_SIZE;
+import static org.apache.phoenix.query.QueryServices.RENEW_LEASE_THRESHOLD_MILLISECONDS;
+import static org.apache.phoenix.query.QueryServices.RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDriver;
+import org.apache.phoenix.query.ConnectionQueryServices.Feature;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.junit.AfterClass;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class ScannerLeaseRenewalIT {
+ private static HBaseTestingUtility hbaseTestUtil;
+ private static String zkQuorum;
+ // If tests are failing because of scanner expiration errors in mini cluster startup,
+ // increase this timeout. It would end up increasing the duration the tests run too, though.
+ private static final long LEASE_TIMEOUT_PERIOD_MILLIS =
+ HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD / 10;
+ private static String url;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+ hbaseTestUtil = new HBaseTestingUtility(conf);
+ conf.setLong(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, LEASE_TIMEOUT_PERIOD_MILLIS);
+ hbaseTestUtil.startMiniCluster();
+ // establish url and quorum. Need to use PhoenixDriver and not PhoenixTestDriver
+ zkQuorum = "localhost:" + hbaseTestUtil.getZkCluster().getClientPort();
+ url = PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum;
+
+ Properties driverProps = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ driverProps.put(RENEW_LEASE_THREAD_POOL_SIZE, Long.toString(4));
+
+ // if this property is false, tests will fail with UnknownScannerException errors.
+ driverProps.put(RENEW_LEASE_ENABLED, Boolean.toString(true));
+
+ driverProps.put(RENEW_LEASE_THRESHOLD_MILLISECONDS,
+ Long.toString(LEASE_TIMEOUT_PERIOD_MILLIS / 2));
+ driverProps.put(RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS,
+ Long.toString(LEASE_TIMEOUT_PERIOD_MILLIS / 4));
+ driverProps.put(HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
+ Long.toString(LEASE_TIMEOUT_PERIOD_MILLIS));
+ // use round robin iterator
+ driverProps.put(FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.toString(false));
+ DriverManager.registerDriver(PhoenixDriver.INSTANCE);
+ try (PhoenixConnection phxConn = DriverManager.getConnection(url, driverProps).unwrap(PhoenixConnection.class)) {
+ // run test methods only if we are at the hbase version that supports lease renewal.
+ Assume.assumeTrue(phxConn.getQueryServices().supportsFeature(Feature.RENEW_LEASE));
+ }
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ try {
+ DriverManager.deregisterDriver(PhoenixDriver.INSTANCE);
+ } finally {
+ hbaseTestUtil.shutdownMiniCluster();
+ }
+ }
+
+ @Test
+ public void testRenewLeasePreventsSelectQueryFromFailing() throws Exception {
+ String tableName = "testRenewLeasePreventsSelectQueryFromFailing";
+ int numRecords = 5;
+ try (Connection conn = DriverManager.getConnection(url)) {
+ conn.createStatement().execute(
+ "CREATE TABLE " + tableName + " (PK1 INTEGER NOT NULL PRIMARY KEY, KV1 VARCHAR)");
+ int i = 0;
+ String upsert = "UPSERT INTO " + tableName + " VALUES (?, ?)";
+ Random random = new Random();
+ PreparedStatement stmt = conn.prepareStatement(upsert);
+ while (i < numRecords) {
+ stmt.setInt(1, random.nextInt());
+ stmt.setString(2, "KV" + random.nextInt());
+ stmt.executeUpdate();
+ i++;
+ }
+ conn.commit();
+ }
+
+ try (PhoenixConnection phxConn =
+ DriverManager.getConnection(url).unwrap(PhoenixConnection.class)) {
+ String sql = "SELECT * FROM " + tableName;
+ // at every next call wait for this period. This will cause lease to expire.
+ long delayOnNext = 2 * LEASE_TIMEOUT_PERIOD_MILLIS;
+ phxConn.setTableResultIteratorFactory(new DelayedTableResultIteratorFactory(delayOnNext));
+ Statement s = phxConn.createStatement();
+ s.setFetchSize(2);
+ ResultSet rs = s.executeQuery(sql);
+ int count = 0;
+ while (rs.next()) {
+ count++;
+ }
+ assertEquals(numRecords, count);
+ }
+ }
+
+ @Test
+ public void testRenewLeasePreventsUpsertSelectFromFailing() throws Exception {
+ String table1 = "testRenewLeasePreventsUpsertSelectFromFailing";
+ String table2 = "testRenewLeasePreventsUpsertSelectFromFailing2";
+
+ try (Connection conn = DriverManager.getConnection(url)) {
+ conn.createStatement().execute(
+ "CREATE TABLE " + table1 + " (PK1 INTEGER NOT NULL PRIMARY KEY, KV1 VARCHAR)");
+ conn.createStatement().execute(
+ "CREATE TABLE " + table2 + " (PK1 INTEGER NOT NULL PRIMARY KEY, KV1 VARCHAR)");
+ int numRecords = 5;
+ int i = 0;
+ String upsert = "UPSERT INTO " + table1 + " VALUES (?, ?)";
+ Random random = new Random();
+ PreparedStatement stmt = conn.prepareStatement(upsert);
+ while (i < numRecords) {
+ stmt.setInt(1, random.nextInt());
+ stmt.setString(2, "KV" + random.nextInt());
+ stmt.executeUpdate();
+ i++;
+ }
+ conn.commit();
+ }
+
+ try (PhoenixConnection phxConn =
+ DriverManager.getConnection(url).unwrap(PhoenixConnection.class)) {
+ String upsertSelect = "UPSERT INTO " + table2 + " SELECT PK1, KV1 FROM " + table1;
+ // at every next call wait for this period. This will cause lease to expire.
+ long delayAfterInit = 2 * LEASE_TIMEOUT_PERIOD_MILLIS;
+ phxConn.setTableResultIteratorFactory(new DelayedTableResultIteratorFactory(
+ delayAfterInit));
+ Statement s = phxConn.createStatement();
+ s.setFetchSize(2);
+ s.executeUpdate(upsertSelect);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 715e7c4..519b38f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -788,7 +788,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
abstract protected String getName();
abstract protected void submitWork(List<List<Scan>> nestedScans, List<List<Pair<Scan,Future<PeekingResultIterator>>>> nestedFutures,
- Queue<PeekingResultIterator> allIterators, int estFlattenedSize);
+ Queue<PeekingResultIterator> allIterators, int estFlattenedSize) throws SQLException;
@Override
public int size() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java
index ef8c4a6..a17258a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ChunkedResultIterator.java
@@ -126,8 +126,9 @@ public class ChunkedResultIterator implements PeekingResultIterator {
scan.setStartRow(ByteUtil.copyKeyBytesIfNecessary(lastKey));
if (logger.isDebugEnabled()) logger.debug(LogUtil.addCustomAnnotations("Get next chunked result iterator over " + tableRef.getTable().getName().getString() + " with " + scan, ScanUtil.getCustomAnnotations(scan)));
String tableName = tableRef.getTable().getPhysicalName().getString();
+ long renewLeaseThreshold = context.getConnection().getQueryServices().getRenewLeaseThresholdMilliSeconds();
ResultIterator singleChunkResultIterator = new SingleChunkResultIterator(
- new TableResultIterator(mutationState, tableRef, scan, context.getReadMetricsQueue().allotMetric(SCAN_BYTES, tableName)), chunkSize);
+ new TableResultIterator(mutationState, tableRef, scan, context.getReadMetricsQueue().allotMetric(SCAN_BYTES, tableName), renewLeaseThreshold), chunkSize);
resultIterator = delegateIteratorFactory.newIterator(context, singleChunkResultIterator, scan, tableName);
}
return resultIterator;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/iterate/DefaultTableResultIteratorFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/DefaultTableResultIteratorFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/DefaultTableResultIteratorFactory.java
new file mode 100644
index 0000000..6a0c78f
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/DefaultTableResultIteratorFactory.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.iterate;
+
+import java.sql.SQLException;
+
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.phoenix.compile.StatementContext;
+import org.apache.phoenix.monitoring.CombinableMetric;
+import org.apache.phoenix.schema.TableRef;
+
+public class DefaultTableResultIteratorFactory implements TableResultIteratorFactory {
+
+ @Override
+ public TableResultIterator newIterator(StatementContext context, TableRef tableRef, Scan scan,
+ CombinableMetric scanMetrics, long renewLeaseThreshold) throws SQLException {
+ return new TableResultIterator(context.getConnection().getMutationState(), tableRef, scan, scanMetrics, renewLeaseThreshold);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
index 265831c..64c97c7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
@@ -67,7 +67,7 @@ public class ParallelIterators extends BaseResultIterators {
@Override
protected void submitWork(List<List<Scan>> nestedScans, List<List<Pair<Scan,Future<PeekingResultIterator>>>> nestedFutures,
- final Queue<PeekingResultIterator> allIterators, int estFlattenedSize) {
+ final Queue<PeekingResultIterator> allIterators, int estFlattenedSize) throws SQLException {
// Pre-populate nestedFutures lists so that we can shuffle the scans
// and add the future to the right nested list. By shuffling the scans
// we get better utilization of the cluster since our thread executor
@@ -93,24 +93,25 @@ public class ParallelIterators extends BaseResultIterators {
int numScans = scanLocations.size();
context.getOverallQueryMetrics().updateNumParallelScans(numScans);
GLOBAL_NUM_PARALLEL_SCANS.update(numScans);
+ final long renewLeaseThreshold = context.getConnection().getQueryServices().getRenewLeaseThresholdMilliSeconds();
for (ScanLocator scanLocation : scanLocations) {
final Scan scan = scanLocation.getScan();
final CombinableMetric scanMetrics = readMetrics.allotMetric(MetricType.SCAN_BYTES, physicalTableName);
final TaskExecutionMetricsHolder taskMetrics = new TaskExecutionMetricsHolder(readMetrics, physicalTableName);
+ final TableResultIterator tableResultItr = context.getConnection().getTableResultIteratorFactory().newIterator(context, tableRef, scan, scanMetrics, renewLeaseThreshold);
+ context.getConnection().addIterator(tableResultItr);
Future<PeekingResultIterator> future = executor.submit(Tracing.wrap(new JobCallable<PeekingResultIterator>() {
@Override
public PeekingResultIterator call() throws Exception {
long startTime = System.currentTimeMillis();
- ResultIterator scanner = new TableResultIterator(mutationState, tableRef, scan, scanMetrics);
+ tableResultItr.initScanner();
if (logger.isDebugEnabled()) {
logger.debug(LogUtil.addCustomAnnotations("Id: " + scanId + ", Time: " + (System.currentTimeMillis() - startTime) + "ms, Scan: " + scan, ScanUtil.getCustomAnnotations(scan)));
}
- PeekingResultIterator iterator = iteratorFactory.newIterator(context, scanner, scan, physicalTableName);
-
+ PeekingResultIterator iterator = iteratorFactory.newIterator(context, tableResultItr, scan, physicalTableName);
// Fill the scanner's cache. This helps reduce latency since we are parallelizing the I/O needed.
iterator.peek();
-
allIterators.add(iterator);
return iterator;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/iterate/ScanningResultIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ScanningResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ScanningResultIterator.java
index 604e1a7..7f865ed 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ScanningResultIterator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ScanningResultIterator.java
@@ -38,12 +38,12 @@ import org.apache.phoenix.util.ServerUtil;
public class ScanningResultIterator implements ResultIterator {
private final ResultScanner scanner;
private final CombinableMetric scanMetrics;
-
+
public ScanningResultIterator(ResultScanner scanner, CombinableMetric scanMetrics) {
this.scanner = scanner;
this.scanMetrics = scanMetrics;
}
-
+
@Override
public void close() throws SQLException {
scanner.close();
@@ -70,11 +70,11 @@ public class ScanningResultIterator implements ResultIterator {
public void explain(List<String> planSteps) {
}
- @Override
- public String toString() {
- return "ScanningResultIterator [scanner=" + scanner + "]";
- }
-
+ @Override
+ public String toString() {
+ return "ScanningResultIterator [scanner=" + scanner + "]";
+ }
+
private void calculateScanSize(Result result) {
if (GlobalClientMetrics.isMetricsEnabled() || scanMetrics != NoOpRequestMetric.INSTANCE) {
if (result != null) {
@@ -89,4 +89,9 @@ public class ScanningResultIterator implements ResultIterator {
}
}
}
+
+
+ public ResultScanner getScanner() {
+ return scanner;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
index e0936b2..a221ba2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
@@ -29,7 +29,7 @@ import java.util.concurrent.Future;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.compile.QueryPlan;
-import org.apache.phoenix.iterate.TableResultIterator.ScannerCreation;
+import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.job.JobManager.JobCallable;
import org.apache.phoenix.monitoring.TaskExecutionMetricsHolder;
import org.apache.phoenix.trace.util.Tracing;
@@ -75,12 +75,15 @@ public class SerialIterators extends BaseResultIterators {
overallScan.setStopRow(lastScan.getStopRow());
final String tableName = tableRef.getTable().getPhysicalName().getString();
final TaskExecutionMetricsHolder taskMetrics = new TaskExecutionMetricsHolder(context.getReadMetricsQueue(), tableName);
+ final PhoenixConnection conn = context.getConnection();
+ final long renewLeaseThreshold = conn.getQueryServices().getRenewLeaseThresholdMilliSeconds();
Future<PeekingResultIterator> future = executor.submit(Tracing.wrap(new JobCallable<PeekingResultIterator>() {
@Override
public PeekingResultIterator call() throws Exception {
List<PeekingResultIterator> concatIterators = Lists.newArrayListWithExpectedSize(scans.size());
for (final Scan scan : scans) {
- ResultIterator scanner = new TableResultIterator(mutationState, tableRef, scan, context.getReadMetricsQueue().allotMetric(SCAN_BYTES, tableName), ScannerCreation.DELAYED);
+ TableResultIterator scanner = new TableResultIterator(mutationState, tableRef, scan, context.getReadMetricsQueue().allotMetric(SCAN_BYTES, tableName), renewLeaseThreshold);
+ conn.addIterator(scanner);
concatIterators.add(iteratorFactory.newIterator(context, scanner, scan, tableName));
}
PeekingResultIterator concatIterator = ConcatResultIterator.newIterator(concatIterators);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java
index 7d07a4a..d97a535 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java
@@ -17,22 +17,31 @@
*/
package org.apache.phoenix.iterate;
+import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.CLOSED;
+import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.NOT_RENEWED;
+import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.RENEWED;
+import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.THRESHOLD_NOT_REACHED;
+import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.UNINITIALIZED;
+
import java.io.IOException;
import java.sql.SQLException;
import java.util.List;
+import javax.annotation.concurrent.GuardedBy;
+
+import org.apache.hadoop.hbase.client.AbstractClientScanner;
import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.phoenix.execute.MutationState;
-import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.monitoring.CombinableMetric;
+import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.util.Closeables;
import org.apache.phoenix.util.ServerUtil;
+import com.google.common.annotations.VisibleForTesting;
+
/**
*
@@ -43,84 +52,116 @@ import org.apache.phoenix.util.ServerUtil;
* @since 0.1
*/
public class TableResultIterator implements ResultIterator {
- public enum ScannerCreation {IMMEDIATE, DELAYED};
-
private final Scan scan;
private final HTableInterface htable;
- private volatile ResultIterator delegate;
private final CombinableMetric scanMetrics;
-
- public TableResultIterator(MutationState mutationState, Scan scan, TableRef tableRef, CombinableMetric scanMetrics) throws SQLException {
- this(mutationState, tableRef, scan, scanMetrics);
- }
+ private static final ResultIterator UNINITIALIZED_SCANNER = ResultIterator.EMPTY_ITERATOR;
+ private final long renewLeaseThreshold;
- /*
- * Delay the creation of the underlying HBase ResultScanner if creationMode is DELAYED.
- * Though no rows are returned when the scanner is created, it still makes several RPCs
- * to open the scanner. In queries run serially (i.e. SELECT ... LIMIT 1), we do not
- * want to be hit with this cost when it's likely we'll never execute those scanners.
- */
- private ResultIterator getDelegate(boolean isClosing) throws SQLException {
- ResultIterator delegate = this.delegate;
- if (delegate == null) {
- synchronized (this) {
- delegate = this.delegate;
- if (delegate == null) {
- try {
- this.delegate = delegate = isClosing ? ResultIterator.EMPTY_ITERATOR : new ScanningResultIterator(htable.getScanner(scan), scanMetrics);
- } catch (IOException e) {
- Closeables.closeQuietly(htable);
- throw ServerUtil.parseServerException(e);
- }
- }
- }
- }
- return delegate;
- }
-
- public TableResultIterator(MutationState mutationState, TableRef tableRef, Scan scan, CombinableMetric scanMetrics) throws SQLException {
- this(mutationState, tableRef, scan, scanMetrics, ScannerCreation.IMMEDIATE);
+ @GuardedBy("this")
+ private ResultIterator scanIterator;
+
+ @GuardedBy("this")
+ private boolean closed = false;
+
+ @GuardedBy("this")
+ private long renewLeaseTime = 0;
+
+ @VisibleForTesting // Exposed for testing. DON'T USE ANYWHERE ELSE!
+ TableResultIterator() {
+ this.scanMetrics = null;
+ this.renewLeaseThreshold = 0;
+ this.htable = null;
+ this.scan = null;
}
- public TableResultIterator(MutationState mutationState, TableRef tableRef, Scan scan, CombinableMetric scanMetrics, ScannerCreation creationMode) throws SQLException {
+ public static enum RenewLeaseStatus {
+ RENEWED, CLOSED, UNINITIALIZED, THRESHOLD_NOT_REACHED, NOT_RENEWED
+ };
+
+
+ public TableResultIterator(MutationState mutationState, TableRef tableRef, Scan scan, CombinableMetric scanMetrics, long renewLeaseThreshold) throws SQLException {
this.scan = scan;
this.scanMetrics = scanMetrics;
PTable table = tableRef.getTable();
htable = mutationState.getHTable(table);
- if (creationMode == ScannerCreation.IMMEDIATE) {
- getDelegate(false);
- }
+ this.scanIterator = UNINITIALIZED_SCANNER;
+ this.renewLeaseThreshold = renewLeaseThreshold;
}
@Override
- public void close() throws SQLException {
+ public synchronized void close() throws SQLException {
+ closed = true; // ok to say closed even if the below code throws an exception
try {
- getDelegate(true).close();
+ scanIterator.close();
} finally {
try {
+ scanIterator = UNINITIALIZED_SCANNER;
htable.close();
} catch (IOException e) {
throw ServerUtil.parseServerException(e);
}
}
}
+
+ @Override
+ public synchronized Tuple next() throws SQLException {
+ initScanner();
+ Tuple t = scanIterator.next();
+ return t;
+ }
+
+ public synchronized void initScanner() throws SQLException {
+ if (closed) {
+ return;
+ }
+ ResultIterator delegate = this.scanIterator;
+ if (delegate == UNINITIALIZED_SCANNER) {
+ try {
+ this.scanIterator =
+ new ScanningResultIterator(htable.getScanner(scan), scanMetrics);
+ } catch (IOException e) {
+ Closeables.closeQuietly(htable);
+ throw ServerUtil.parseServerException(e);
+ }
+ }
+ }
@Override
- public Tuple next() throws SQLException {
- return getDelegate(false).next();
+ public String toString() {
+ return "TableResultIterator [htable=" + htable + ", scan=" + scan + "]";
+ }
+
+ public synchronized RenewLeaseStatus renewLease() {
+ if (closed) {
+ return CLOSED;
+ }
+ if (scanIterator == UNINITIALIZED_SCANNER) {
+ return UNINITIALIZED;
+ }
+ long delay = now() - renewLeaseTime;
+ if (delay < renewLeaseThreshold) {
+ return THRESHOLD_NOT_REACHED;
+ }
+ if (scanIterator instanceof ScanningResultIterator
+ && ((ScanningResultIterator)scanIterator).getScanner() instanceof AbstractClientScanner) {
+ // Need this explicit cast because HBase's ResultScanner doesn't have this method exposed.
+ boolean leaseRenewed = ((AbstractClientScanner)((ScanningResultIterator)scanIterator).getScanner()).renewLease();
+ if (leaseRenewed) {
+ renewLeaseTime = now();
+ return RENEWED;
+ }
+ }
+ return NOT_RENEWED;
+ }
+
+ private static long now() {
+ return System.currentTimeMillis();
}
@Override
public void explain(List<String> planSteps) {
- try {
- getDelegate(false).explain(planSteps);
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
+ scanIterator.explain(planSteps);
}
- @Override
- public String toString() {
- return "TableResultIterator [htable=" + htable + ", scan=" + scan + "]";
- }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIteratorFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIteratorFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIteratorFactory.java
new file mode 100644
index 0000000..083daf1
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableResultIteratorFactory.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.iterate;
+
+import java.sql.SQLException;
+
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.phoenix.compile.StatementContext;
+import org.apache.phoenix.monitoring.CombinableMetric;
+import org.apache.phoenix.schema.TableRef;
+
+public interface TableResultIteratorFactory {
+ public TableResultIterator newIterator(StatementContext context, TableRef tableRef, Scan scan, CombinableMetric scanMetrics, long renewLeaseThreshold) throws SQLException;
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
index d547fb4..b7bfc34 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
@@ -108,8 +108,14 @@ import com.google.common.collect.ImmutableMap.Builder;
import com.google.common.collect.Lists;
import co.cask.tephra.TransactionContext;
-
-
+import java.lang.ref.WeakReference;
+import java.util.concurrent.LinkedBlockingQueue;
+import javax.annotation.Nonnull;
+import org.apache.phoenix.iterate.DefaultTableResultIteratorFactory;
+import org.apache.phoenix.iterate.TableResultIterator;
+import org.apache.phoenix.iterate.TableResultIteratorFactory;
+import com.google.common.annotations.VisibleForTesting;
+import static com.google.common.base.Preconditions.checkNotNull;
/**
*
* JDBC Connection implementation of Phoenix.
@@ -141,7 +147,7 @@ public class PhoenixConnection implements Connection, MetaDataMutated, SQLClosea
private final String timestampPattern;
private int statementExecutionCounter;
private TraceScope traceScope = null;
- private boolean isClosed = false;
+ private volatile boolean isClosed = false;
private Sampler<?> sampler;
private boolean readOnly = false;
private Consistency consistency = Consistency.STRONG;
@@ -150,6 +156,9 @@ public class PhoenixConnection implements Connection, MetaDataMutated, SQLClosea
private final boolean isDescVarLengthRowKeyUpgrade;
private ParallelIteratorFactory parallelIteratorFactory;
+ private final LinkedBlockingQueue<WeakReference<TableResultIterator>> scannerQueue;
+ private TableResultIteratorFactory tableResultIteratorFactory;
+
static {
Tracing.addTraceMetricsSource();
}
@@ -278,6 +287,8 @@ public class PhoenixConnection implements Connection, MetaDataMutated, SQLClosea
// setup tracing, if its enabled
this.sampler = Tracing.getConfiguredSampler(this);
this.customTracingAnnotations = getImmutableCustomTracingAnnotations();
+ this.scannerQueue = new LinkedBlockingQueue<>();
+ this.tableResultIteratorFactory = new DefaultTableResultIteratorFactory();
}
private static void checkScn(Long scnParam) throws SQLException {
@@ -1002,4 +1013,27 @@ public class PhoenixConnection implements Connection, MetaDataMutated, SQLClosea
public void setIteratorFactory(ParallelIteratorFactory parallelIteratorFactory) {
this.parallelIteratorFactory = parallelIteratorFactory;
}
+
+ public void addIterator(@Nonnull TableResultIterator itr) {
+ if (services.supportsFeature(ConnectionQueryServices.Feature.RENEW_LEASE)) {
+ checkNotNull(itr);
+ scannerQueue.add(new WeakReference<TableResultIterator>(itr));
+ }
+ }
+
+ public LinkedBlockingQueue<WeakReference<TableResultIterator>> getScanners() {
+ return scannerQueue;
+ }
+
+ @VisibleForTesting
+ @Nonnull
+ public TableResultIteratorFactory getTableResultIteratorFactory() {
+ return tableResultIteratorFactory;
+ }
+
+ @VisibleForTesting
+ public void setTableResultIteratorFactory(TableResultIteratorFactory factory) {
+ checkNotNull(factory);
+ this.tableResultIteratorFactory = factory;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index a4748b3..21bd591 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -288,7 +288,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
private final ResultSet emptyResultSet;
public static final int MAX_LOCAL_SI_VERSION_DISALLOW = VersionUtil.encodeVersion("0", "98", "8");
public static final int MIN_LOCAL_SI_VERSION_DISALLOW = VersionUtil.encodeVersion("0", "98", "6");
-
+ public static final int MIN_RENEW_LEASE_VERSION = VersionUtil.encodeVersion("1", "0", "4");
+
// Version below which we should turn off essential column family.
public static final int ESSENTIAL_FAMILY_VERSION_THRESHOLD = VersionUtil.encodeVersion("0", "94", "7");
// Version below which we should disallow usage of mutable secondary indexing.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java
index d2f91b3..2df76fc 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java
@@ -107,9 +107,10 @@ public class PhoenixRecordReader<T extends DBWritable> extends RecordReader<Null
StatementContext ctx = queryPlan.getContext();
ReadMetricQueue readMetrics = ctx.getReadMetricsQueue();
String tableName = queryPlan.getTableRef().getTable().getPhysicalName().getString();
+ long renewScannerLeaseThreshold = queryPlan.getContext().getConnection().getQueryServices().getRenewLeaseThresholdMilliSeconds();
for (Scan scan : scans) {
final TableResultIterator tableResultIterator = new TableResultIterator(queryPlan.getContext().getConnection().getMutationState(),
- queryPlan.getTableRef(), scan, readMetrics.allotMetric(SCAN_BYTES, tableName));
+ queryPlan.getTableRef(), scan, readMetrics.allotMetric(SCAN_BYTES, tableName), renewScannerLeaseThreshold);
PeekingResultIterator peekingResultIterator = LookAheadResultIterator.wrap(tableResultIterator);
iterators.add(peekingResultIterator);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index d839fa3..25d7ff4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -23,6 +23,8 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import co.cask.tephra.TransactionSystemClient;
+
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -44,8 +46,6 @@ import org.apache.phoenix.schema.SequenceAllocation;
import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.stats.PTableStats;
-import co.cask.tephra.TransactionSystemClient;
-
public interface ConnectionQueryServices extends QueryServices, MetaDataMutated {
public static final int INITIAL_META_DATA_TABLE_CAPACITY = 100;
@@ -108,7 +108,7 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
*/
public KeyValueBuilder getKeyValueBuilder();
- public enum Feature {LOCAL_INDEX};
+ public enum Feature {LOCAL_INDEX, RENEW_LEASE};
public boolean supportsFeature(Feature feature);
public String getUserName();
@@ -120,4 +120,6 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
public int getSequenceSaltBuckets();
TransactionSystemClient getTransactionSystemClient();
+ public long getRenewLeaseThresholdMilliSeconds();
+ public boolean isRenewingLeasesEnabled();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index daa7809..0472d3a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -16,16 +16,21 @@
* limitations under the License.
*/
package org.apache.phoenix.query;
-
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.hadoop.hbase.HColumnDescriptor.TTL;
import static org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_MAJOR_VERSION;
import static org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_MINOR_VERSION;
import static org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_PATCH_NUMBER;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES;
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_DROP_METADATA;
+import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RENEW_LEASE_ENABLED;
+import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RENEW_LEASE_THREAD_POOL_SIZE;
+import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RENEW_LEASE_THRESHOLD_MILLISECONDS;
+import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS;
import static org.apache.phoenix.util.UpgradeUtil.upgradeTo4_5_0;
import java.io.IOException;
+import java.lang.ref.WeakReference;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
@@ -35,17 +40,31 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
+import java.util.Random;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.concurrent.GuardedBy;
+import co.cask.tephra.TransactionSystemClient;
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.distributed.PooledClientProvider;
+import co.cask.tephra.distributed.TransactionServiceClient;
+import co.cask.tephra.hbase10.coprocessor.TransactionProcessor;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
@@ -117,6 +136,8 @@ import org.apache.phoenix.hbase.index.util.VersionUtil;
import org.apache.phoenix.index.PhoenixIndexBuilder;
import org.apache.phoenix.index.PhoenixIndexCodec;
import org.apache.phoenix.index.PhoenixTransactionalIndexer;
+import org.apache.phoenix.iterate.TableResultIterator;
+import org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo;
@@ -173,27 +194,17 @@ import org.apache.twill.zookeeper.ZKClients;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.distributed.PooledClientProvider;
-import co.cask.tephra.distributed.TransactionServiceClient;
-import co.cask.tephra.hbase10.coprocessor.TransactionProcessor;
-
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Throwables;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.distributed.PooledClientProvider;
-import co.cask.tephra.distributed.TransactionServiceClient;
-import co.cask.tephra.hbase10.coprocessor.TransactionProcessor;
-
public class ConnectionQueryServicesImpl extends DelegateQueryServices implements ConnectionQueryServices {
private static final Logger logger = LoggerFactory.getLogger(ConnectionQueryServicesImpl.class);
private static final int INITIAL_CHILD_SERVICES_CAPACITY = 100;
@@ -235,6 +246,14 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
// setting this member variable guarded by "connectionCountLock"
private volatile ConcurrentMap<SequenceKey,Sequence> sequenceMap = Maps.newConcurrentMap();
private KeyValueBuilder kvBuilder;
+ private final int renewLeaseTaskFrequency;
+ private final int renewLeasePoolSize;
+ private final int renewLeaseThreshold;
+ // List of queues instead of a single queue to provide reduced contention via lock striping
+ private final List<LinkedBlockingQueue<WeakReference<PhoenixConnection>>> connectionQueues;
+ private ScheduledExecutorService renewLeaseExecutor;
+ private final boolean renewLeaseEnabled;
+
private static interface FeatureSupported {
boolean isSupported(ConnectionQueryServices services);
@@ -247,7 +266,14 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
int hbaseVersion = services.getLowestClusterHBaseVersion();
return hbaseVersion < PhoenixDatabaseMetaData.MIN_LOCAL_SI_VERSION_DISALLOW || hbaseVersion > PhoenixDatabaseMetaData.MAX_LOCAL_SI_VERSION_DISALLOW;
}
- });
+ },
+ Feature.RENEW_LEASE, new FeatureSupported() {
+ @Override
+ public boolean isSupported(ConnectionQueryServices services) {
+ int hbaseVersion = services.getLowestClusterHBaseVersion();
+ return hbaseVersion >= PhoenixDatabaseMetaData.MIN_RENEW_LEASE_VERSION;
+ }
+ });
private PMetaData newEmptyMetaData() {
long maxSizeBytes = props.getLong(QueryServices.MAX_CLIENT_METADATA_CACHE_SIZE_ATTRIB,
@@ -301,6 +327,16 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
.expireAfterWrite(halfStatsUpdateFreq, TimeUnit.MILLISECONDS)
.build();
this.returnSequenceValues = props.getBoolean(QueryServices.RETURN_SEQUENCE_VALUES_ATTRIB, QueryServicesOptions.DEFAULT_RETURN_SEQUENCE_VALUES);
+ this.renewLeaseEnabled = config.getBoolean(RENEW_LEASE_ENABLED, DEFAULT_RENEW_LEASE_ENABLED);
+ this.renewLeasePoolSize = config.getInt(RENEW_LEASE_THREAD_POOL_SIZE, DEFAULT_RENEW_LEASE_THREAD_POOL_SIZE);
+ this.renewLeaseThreshold = config.getInt(RENEW_LEASE_THRESHOLD_MILLISECONDS, DEFAULT_RENEW_LEASE_THRESHOLD_MILLISECONDS);
+ this.renewLeaseTaskFrequency = config.getInt(RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS, DEFAULT_RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS);
+ List<LinkedBlockingQueue<WeakReference<PhoenixConnection>>> list = Lists.newArrayListWithCapacity(renewLeasePoolSize);
+ for (int i = 0; i < renewLeasePoolSize; i++) {
+ LinkedBlockingQueue<WeakReference<PhoenixConnection>> queue = new LinkedBlockingQueue<WeakReference<PhoenixConnection>>();
+ list.add(queue);
+ }
+ connectionQueues = ImmutableList.copyOf(list);
}
@Override
@@ -415,6 +451,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
} finally {
try {
childServices.clear();
+ if (renewLeaseExecutor != null) {
+ renewLeaseExecutor.shutdownNow();
+ }
synchronized (latestMetaDataLock) {
latestMetaData = null;
latestMetaDataLock.notifyAll();
@@ -2244,6 +2283,17 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
String tableName, long timestamp, String columns) throws SQLException {
return addColumn(oldMetaConnection, tableName, timestamp, columns, true);
}
+
+ private static class RenewLeaseThreadFactory implements ThreadFactory {
+ private final AtomicInteger counter = new AtomicInteger(1);
+
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r);
+ t.setName("Phoenix-Scanner-Renewlease-" + counter.getAndIncrement());
+ return t;
+ }
+ }
@Override
public void init(final String url, final Properties props) throws SQLException {
@@ -2441,8 +2491,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
} catch (NewerTableAlreadyExistsException e) {
} catch (TableAlreadyExistsException e) {
}
-
-
+ scheduleRenewLeaseTasks();
} catch (Exception e) {
if (e instanceof SQLException) {
initializationException = (SQLException)e;
@@ -2479,6 +2528,18 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
+ private void scheduleRenewLeaseTasks() {
+ if (isRenewingLeasesEnabled()) {
+ renewLeaseExecutor =
+ Executors.newScheduledThreadPool(renewLeasePoolSize,
+ new RenewLeaseThreadFactory());
+ for (LinkedBlockingQueue<WeakReference<PhoenixConnection>> q : connectionQueues) {
+ renewLeaseExecutor.scheduleAtFixedRate(new RenewLeaseTask(q), 0,
+ renewLeaseTaskFrequency, TimeUnit.MILLISECONDS);
+ }
+ }
+ }
+
private static int getSaltBuckets(TableAlreadyExistsException e) {
PTable table = e.getTable();
Integer sequenceSaltBuckets = table == null ? null : table.getBucketNum();
@@ -2957,12 +3018,17 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
@Override
public void addConnection(PhoenixConnection connection) throws SQLException {
+ connectionQueues.get(getQueueIndex(connection)).add(new WeakReference<PhoenixConnection>(connection));
if (returnSequenceValues) {
synchronized (connectionCountLock) {
connectionCount++;
}
}
}
+
+ private int getQueueIndex(PhoenixConnection conn) {
+ return ThreadLocalRandom.current().nextInt(renewLeasePoolSize);
+ }
@Override
public void removeConnection(PhoenixConnection connection) throws SQLException {
@@ -3153,4 +3219,102 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}, PhoenixDatabaseMetaData.SYSTEM_FUNCTION_NAME_BYTES);
return result;
}
+
+ @VisibleForTesting
+ static class RenewLeaseTask implements Runnable {
+
+ private final LinkedBlockingQueue<WeakReference<PhoenixConnection>> connectionsQueue;
+ private final Random random = new Random();
+ private static final int MAX_WAIT_TIME = 1000;
+
+ RenewLeaseTask(LinkedBlockingQueue<WeakReference<PhoenixConnection>> queue) {
+ this.connectionsQueue = queue;
+ }
+
+ private void waitForRandomDuration() {
+ try {
+ new CountDownLatch(1).await(random.nextInt(MAX_WAIT_TIME), MILLISECONDS);
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ logger.warn("Exception encountered while waiting before renewing leases: " + ex);
+ }
+ }
+
+ @Override
+ public void run() {
+ int numConnections = connectionsQueue.size();
+ boolean wait = true;
+ // We keep adding items to the end of the queue. So to stop the loop, iterate only up to
+ // whatever the current count is.
+ while (numConnections > 0) {
+ if (wait) {
+ // wait for some random duration to prevent all threads from renewing lease at
+ // the same time.
+ waitForRandomDuration();
+ wait = false;
+ }
+ // It is guaranteed that this poll won't hang indefinitely because this is the
+ // only thread that removes items from the queue.
+ WeakReference<PhoenixConnection> connRef = connectionsQueue.poll();
+ PhoenixConnection conn = connRef.get();
+ try {
+ if (conn != null && !conn.isClosed()) {
+ LinkedBlockingQueue<WeakReference<TableResultIterator>> scannerQueue =
+ conn.getScanners();
+ // We keep adding items to the end of the queue. So to stop the loop,
+ // iterate only up to whatever the current count is.
+ int numScanners = scannerQueue.size();
+ int renewed = 0;
+ long start = System.currentTimeMillis();
+ while (numScanners > 0) {
+ WeakReference<TableResultIterator> ref = scannerQueue.poll();
+ TableResultIterator scanningItr = ref.get();
+ if (scanningItr != null) {
+ RenewLeaseStatus status = scanningItr.renewLease();
+ switch (status) {
+ case RENEWED:
+ renewed++;
+ // add it back at the tail
+ scannerQueue.offer(new WeakReference<TableResultIterator>(
+ scanningItr));
+ logger.info("Lease renewed for scanner: " + scanningItr);
+ break;
+ case UNINITIALIZED:
+ case THRESHOLD_NOT_REACHED:
+ // add it back at the tail
+ scannerQueue.offer(new WeakReference<TableResultIterator>(
+ scanningItr));
+ break;
+ // if lease wasn't renewed or scanner was closed, don't add the
+ // scanner back to the queue.
+ case CLOSED:
+ case NOT_RENEWED:
+ break;
+ }
+ }
+ numScanners--;
+ }
+ if (renewed > 0) {
+ logger.info("Renewed leases for " + renewed + " scanner/s in "
+ + (System.currentTimeMillis() - start) + " ms ");
+ }
+ connectionsQueue.offer(connRef);
+ }
+ } catch (Exception e) {
+ logger.warn("Exception encountered when renewing lease: " + e);
+ }
+ numConnections--;
+ }
+ }
+ }
+
+ @Override
+ public long getRenewLeaseThresholdMilliSeconds() {
+ return renewLeaseThreshold;
+ }
+
+ @Override
+ public boolean isRenewingLeasesEnabled() {
+ return supportsFeature(ConnectionQueryServices.Feature.RENEW_LEASE) && renewLeaseEnabled;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index f1ab319..199b010 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -567,4 +567,14 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
throws SQLException {
return new MetaDataMutationResult(MutationCode.FUNCTION_ALREADY_EXISTS, 0, null);
}
+
+ @Override
+ public long getRenewLeaseThresholdMilliSeconds() {
+ return 0;
+ }
+
+ @Override
+ public boolean isRenewingLeasesEnabled() {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index b56ff85..84f3e74 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -265,6 +265,7 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
return getDelegate().getTransactionSystemClient();
}
+ @Override
public MetaDataMutationResult createFunction(List<Mutation> functionData, PFunction function, boolean temporary)
throws SQLException {
return getDelegate().createFunction(functionData, function, temporary);
@@ -293,4 +294,14 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
throws SQLException {
return getDelegate().dropFunction(tableMetadata, ifExists);
}
+
+ @Override
+ public long getRenewLeaseThresholdMilliSeconds() {
+ return getDelegate().getRenewLeaseThresholdMilliSeconds();
+ }
+
+ @Override
+ public boolean isRenewingLeasesEnabled() {
+ return getDelegate().isRenewingLeasesEnabled();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 8f5f9a1..5fd7403 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -196,6 +196,11 @@ public interface QueryServices extends SQLCloseable {
public static final String QUERY_SERVER_DNS_NAMESERVER_ATTRIB = "phoenix.queryserver.dns.nameserver";
public static final String QUERY_SERVER_DNS_INTERFACE_ATTRIB = "phoenix.queryserver.dns.interface";
public static final String QUERY_SERVER_HBASE_SECURITY_CONF_ATTRIB = "hbase.security.authentication";
+
+ public static final String RENEW_LEASE_ENABLED = "phoenix.scanner.lease.renew.enabled";
+ public static final String RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS = "phoenix.scanner.lease.renew.interval";
+ public static final String RENEW_LEASE_THRESHOLD_MILLISECONDS = "phoenix.scanner.lease.threshold";
+ public static final String RENEW_LEASE_THREAD_POOL_SIZE = "phoenix.scanner.lease.pool.size";
/**
* Get executor service used for parallel scans
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index b256ff3..def0a18 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.query;
+import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD;
import static org.apache.phoenix.query.QueryServices.ALLOW_ONLINE_TABLE_SCHEMA_UPDATE;
import static org.apache.phoenix.query.QueryServices.ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE;
import static org.apache.phoenix.query.QueryServices.CALL_QUEUE_PRODUCER_ATTRIB_NAME;
@@ -53,8 +54,12 @@ import static org.apache.phoenix.query.QueryServices.NUM_RETRIES_FOR_SCHEMA_UPDA
import static org.apache.phoenix.query.QueryServices.QUEUE_SIZE_ATTRIB;
import static org.apache.phoenix.query.QueryServices.REGIONSERVER_INFO_PORT_ATTRIB;
import static org.apache.phoenix.query.QueryServices.REGIONSERVER_LEASE_PERIOD_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.RENEW_LEASE_ENABLED;
+import static org.apache.phoenix.query.QueryServices.RENEW_LEASE_THREAD_POOL_SIZE;
+import static org.apache.phoenix.query.QueryServices.RENEW_LEASE_THRESHOLD_MILLISECONDS;
import static org.apache.phoenix.query.QueryServices.ROW_KEY_ORDER_SALTED_TABLE_ATTRIB;
import static org.apache.phoenix.query.QueryServices.RPC_TIMEOUT_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS;
import static org.apache.phoenix.query.QueryServices.RUN_UPDATE_STATS_ASYNC;
import static org.apache.phoenix.query.QueryServices.SCAN_CACHE_SIZE_ATTRIB;
import static org.apache.phoenix.query.QueryServices.SCAN_RESULT_CHUNK_SIZE;
@@ -224,6 +229,13 @@ public class QueryServicesOptions {
// doesn't depend on phoenix-core.
public static final String DEFAULT_QUERY_SERVER_SERIALIZATION = "PROTOBUF";
public static final int DEFAULT_QUERY_SERVER_HTTP_PORT = 8765;
+ public static final boolean DEFAULT_RENEW_LEASE_ENABLED = true;
+ public static final int DEFAULT_RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS =
+ DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD / 2;
+ public static final int DEFAULT_RENEW_LEASE_THRESHOLD_MILLISECONDS =
+ (3 * DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD) / 4;
+ public static final int DEFAULT_RENEW_LEASE_THREAD_POOL_SIZE = 10;
+
@SuppressWarnings("serial")
public static final Set<String> DEFAULT_QUERY_SERVER_SKIP_WORDS = new HashSet<String>() {
{
@@ -233,7 +245,7 @@ public class QueryServicesOptions {
add("credential");
}
};
-
+
private final Configuration config;
private QueryServicesOptions(Configuration config) {
@@ -292,7 +304,11 @@ public class QueryServicesOptions {
.setIfUnset(USE_BYTE_BASED_REGEX_ATTRIB, DEFAULT_USE_BYTE_BASED_REGEX)
.setIfUnset(FORCE_ROW_KEY_ORDER_ATTRIB, DEFAULT_FORCE_ROW_KEY_ORDER)
.setIfUnset(COLLECT_REQUEST_LEVEL_METRICS, DEFAULT_REQUEST_LEVEL_METRICS_ENABLED)
- .setIfUnset(ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE, DEFAULT_ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE);
+ .setIfUnset(ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE, DEFAULT_ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE)
+ .setIfUnset(ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE, DEFAULT_ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE)
+ .setIfUnset(RENEW_LEASE_THRESHOLD_MILLISECONDS, DEFAULT_RENEW_LEASE_THRESHOLD_MILLISECONDS)
+ .setIfUnset(RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS, DEFAULT_RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS)
+ .setIfUnset(RENEW_LEASE_THREAD_POOL_SIZE, DEFAULT_RENEW_LEASE_THREAD_POOL_SIZE);
;
// HBase sets this to 1, so we reset it to something more appropriate.
// Hopefully HBase will change this, because we can't know if a user set
@@ -595,4 +611,9 @@ public class QueryServicesOptions {
config.setBoolean(COMMIT_STATS_ASYNC, flag);
return this;
}
+
+ public QueryServicesOptions setEnableRenewLease(boolean enable) {
+ config.setBoolean(RENEW_LEASE_ENABLED, enable);
+ return this;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index 0c1c71b..3f16065 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -773,7 +773,7 @@ public abstract class BaseTest {
ensureTableCreated(url, tableName, null, null);
}
- protected static void ensureTableCreated(String url, String tableName, byte[][] splits) throws SQLException {
+ public static void ensureTableCreated(String url, String tableName, byte[][] splits) throws SQLException {
ensureTableCreated(url, tableName, splits, null);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
index 215110c..037e8e7 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
@@ -57,6 +57,7 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
public static final String DEFAULT_EXTRA_JDBC_ARGUMENTS = PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PARAM;
private static final boolean DEFAULT_RUN_UPDATE_STATS_ASYNC = false;
private static final boolean DEFAULT_COMMIT_STATS_ASYNC = false;
+ public static final boolean DEFAULT_RENEW_LEASE_ENABLED = false;
/**
@@ -97,7 +98,8 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
.setForceRowKeyOrder(DEFAULT_FORCE_ROWKEY_ORDER)
.setExtraJDBCArguments(DEFAULT_EXTRA_JDBC_ARGUMENTS)
.setRunUpdateStatsAsync(DEFAULT_RUN_UPDATE_STATS_ASYNC)
- .setCommitStatsAsync(DEFAULT_COMMIT_STATS_ASYNC);
+ .setCommitStatsAsync(DEFAULT_COMMIT_STATS_ASYNC)
+ .setEnableRenewLease(DEFAULT_RENEW_LEASE_ENABLED);
}
public QueryServicesTestImpl(ReadOnlyProps defaultProps, ReadOnlyProps overrideProps) {
[42/50] [abbrv] phoenix git commit: PHOENIX-2559 Keep filter for OR
when only some expressions are extracted to stop/stop key
Posted by sa...@apache.org.
PHOENIX-2559 Keep filter for OR when only some expressions are extracted to stop/stop key
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a7788daf
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a7788daf
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a7788daf
Branch: refs/heads/4.x-HBase-1.0
Commit: a7788dafac19cd1ce0bb2d50b972dd9079673b51
Parents: c027e0b
Author: James Taylor <jt...@salesforce.com>
Authored: Mon Jan 4 17:29:40 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Jan 4 17:32:39 2016 -0800
----------------------------------------------------------------------
.../apache/phoenix/end2end/SkipScanQueryIT.java | 20 ++++++++
.../apache/phoenix/compile/WhereOptimizer.java | 52 +++++++++++++++----
.../phoenix/compile/WhereCompilerTest.java | 54 +++++++++++++++++++-
3 files changed, 114 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a7788daf/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java
index 2ade0a4..9f73550 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java
@@ -414,4 +414,24 @@ public class SkipScanQueryIT extends BaseHBaseManagedTimeIT {
conn.close();
}
}
+
+ @Test
+ public void testOrPKWithAndNonPK() throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ try {
+ conn.createStatement().execute("create table bugTable(ID varchar primary key,company varchar)");
+ conn.setAutoCommit(true);
+ conn.createStatement().execute("upsert into bugTable values('i1','c1')");
+ conn.createStatement().execute("upsert into bugTable values('i2','c2')");
+ conn.createStatement().execute("upsert into bugTable values('i3','c3')");
+ ResultSet rs = conn.createStatement().executeQuery("select * from bugTable where ID = 'i1' or (ID = 'i2' and company = 'c3')");
+ assertTrue(rs.next());
+ assertEquals("i1", rs.getString(1));
+ assertEquals("c1", rs.getString(2));
+ assertFalse(rs.next());
+
+ } finally {
+ conn.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a7788daf/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
index 99986a2..0122712 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
@@ -456,6 +456,11 @@ public class WhereOptimizer {
public KeyRange getMinMaxRange() {
return null;
}
+
+ @Override
+ public boolean isPartialExtraction() {
+ return false;
+ }
};
private static boolean isDegenerate(List<KeyRange> keyRanges) {
@@ -643,10 +648,12 @@ public class WhereOptimizer {
KeyRange minMaxRange = KeyRange.EVERYTHING_RANGE;
List<Expression> minMaxExtractNodes = Lists.<Expression>newArrayList();
int initPosition = (table.getBucketNum() ==null ? 0 : 1) + (this.context.getConnection().getTenantId() != null && table.isMultiTenant() ? 1 : 0) + (table.getViewIndexId() == null ? 0 : 1);
+ boolean partialExtraction = andExpression.getChildren().size() != childSlots.size();
for (KeySlots childSlot : childSlots) {
if (childSlot == EMPTY_KEY_SLOTS) {
return EMPTY_KEY_SLOTS;
}
+ partialExtraction |= childSlot.isPartialExtraction();
// FIXME: get rid of this special-cased min/max range now that a key range can span multiple columns
if (childSlot.getMinMaxRange() != null) { // Only set if in initial pk position
// TODO: fix intersectSlots so that it works with RVCs. We'd just need to fill in the leading parts
@@ -688,7 +695,7 @@ public class WhereOptimizer {
// If we have a salt column, skip that slot because
// they'll never be an expression contained by it.
keySlots = keySlots.subList(initPosition, keySlots.size());
- return new MultiKeySlot(keySlots, minMaxRange == KeyRange.EVERYTHING_RANGE ? null : minMaxRange);
+ return new MultiKeySlot(keySlots, minMaxRange == KeyRange.EVERYTHING_RANGE ? null : minMaxRange, partialExtraction);
}
private KeySlots orKeySlots(OrExpression orExpression, List<KeySlots> childSlots) {
@@ -708,7 +715,7 @@ public class WhereOptimizer {
KeySlot theSlot = null;
List<Expression> slotExtractNodes = Lists.<Expression>newArrayList();
int thePosition = -1;
- boolean extractAll = true;
+ boolean partialExtraction = false;
// TODO: Have separate list for single span versus multi span
// For multi-span, we only need to keep a single range.
List<KeyRange> slotRanges = Lists.newArrayList();
@@ -718,6 +725,10 @@ public class WhereOptimizer {
// TODO: can this ever happen and can we safely filter the expression tree?
continue;
}
+ // When we OR together expressions, we can only extract the entire OR expression
+ // if all sub-expressions have been completely extracted. Otherwise, we must
+ // leave the OR as a post filter.
+ partialExtraction |= childSlot.isPartialExtraction();
if (childSlot.getMinMaxRange() != null) {
if (!slotRanges.isEmpty() && thePosition != initialPos) { // ORing together rvc in initial slot with other slots
return null;
@@ -726,9 +737,7 @@ public class WhereOptimizer {
thePosition = initialPos;
for (KeySlot slot : childSlot) {
if (slot != null) {
- List<Expression> extractNodes = slot.getKeyPart().getExtractNodes();
- extractAll &= !extractNodes.isEmpty();
- slotExtractNodes.addAll(extractNodes);
+ slotExtractNodes.addAll(slot.getKeyPart().getExtractNodes());
}
}
} else {
@@ -757,9 +766,7 @@ public class WhereOptimizer {
} else if (thePosition != slot.getPKPosition()) {
return null;
}
- List<Expression> extractNodes = slot.getKeyPart().getExtractNodes();
- extractAll &= !extractNodes.isEmpty();
- slotExtractNodes.addAll(extractNodes);
+ slotExtractNodes.addAll(slot.getKeyPart().getExtractNodes());
slotRanges.addAll(slot.getKeyRanges());
}
}
@@ -790,7 +797,7 @@ public class WhereOptimizer {
minMaxRange = minMaxRange.union(range);
}
if (clearExtracts) {
- extractAll = false;
+ partialExtraction = true;
slotExtractNodes = Collections.emptyList();
}
slotRanges = Collections.emptyList();
@@ -802,7 +809,7 @@ public class WhereOptimizer {
}
return newKeyParts(
theSlot,
- extractAll ? Collections.<Expression>singletonList(orExpression) : slotExtractNodes,
+ partialExtraction ? slotExtractNodes : Collections.<Expression>singletonList(orExpression),
slotRanges.isEmpty() ? EVERYTHING_RANGES : KeyRange.coalesce(slotRanges),
minMaxRange == KeyRange.EMPTY_RANGE ? null : minMaxRange);
}
@@ -1029,6 +1036,17 @@ public class WhereOptimizer {
private static interface KeySlots extends Iterable<KeySlot> {
@Override public Iterator<KeySlot> iterator();
public KeyRange getMinMaxRange();
+ /**
+ * Tracks whether or not the contained KeySlot(s) contain
+ * a slot that includes only a partial extraction of the
+ * involved expressions. For example: (A AND B) in the case
+ * of A being a PK column and B being a KV column, the
+ * KeySlots representing the AND would return true for
+ * isPartialExtraction.
+ * @return true if a partial expression extraction was
+ * done and false otherwise.
+ */
+ public boolean isPartialExtraction();
}
private final class KeySlot {
@@ -1154,10 +1172,12 @@ public class WhereOptimizer {
private static class MultiKeySlot implements KeySlots {
private final List<KeySlot> childSlots;
private final KeyRange minMaxRange;
+ private final boolean partialExtraction;
- private MultiKeySlot(List<KeySlot> childSlots, KeyRange minMaxRange) {
+ private MultiKeySlot(List<KeySlot> childSlots, KeyRange minMaxRange, boolean partialExtraction) {
this.childSlots = childSlots;
this.minMaxRange = minMaxRange;
+ this.partialExtraction = partialExtraction;
}
@Override
@@ -1169,6 +1189,11 @@ public class WhereOptimizer {
public KeyRange getMinMaxRange() {
return minMaxRange;
}
+
+ @Override
+ public boolean isPartialExtraction() {
+ return partialExtraction;
+ }
}
private class SingleKeySlot implements KeySlots {
@@ -1205,6 +1230,11 @@ public class WhereOptimizer {
public KeyRange getMinMaxRange() {
return minMaxRange;
}
+
+ @Override
+ public boolean isPartialExtraction() {
+ return this.slot.getKeyPart().getExtractNodes().isEmpty();
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a7788daf/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereCompilerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereCompilerTest.java
index a1d1440..7315ad6 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereCompilerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereCompilerTest.java
@@ -50,8 +50,11 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.expression.ColumnExpression;
import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.expression.KeyValueColumnExpression;
import org.apache.phoenix.expression.LiteralExpression;
import org.apache.phoenix.expression.RowKeyColumnExpression;
import org.apache.phoenix.expression.function.SubstrFunction;
@@ -62,7 +65,8 @@ import org.apache.phoenix.jdbc.PhoenixPreparedStatement;
import org.apache.phoenix.query.BaseConnectionlessQueryTest;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryConstants;
-import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.ColumnRef;
+import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.RowKeyValueAccessor;
import org.apache.phoenix.schema.SaltingUtil;
import org.apache.phoenix.schema.types.PChar;
@@ -106,6 +110,54 @@ public class WhereCompilerTest extends BaseConnectionlessQueryTest {
}
@Test
+ public void testOrPKWithAndPKAndNotPK() throws SQLException {
+ String query = "select * from bugTable where ID = 'i1' or (ID = 'i2' and company = 'c3')";
+ PhoenixConnection pconn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)).unwrap(PhoenixConnection.class);
+ pconn.createStatement().execute("create table bugTable(ID varchar primary key,company varchar)");
+ PhoenixPreparedStatement pstmt = newPreparedStatement(pconn, query);
+ QueryPlan plan = pstmt.optimizeQuery();
+ Scan scan = plan.getContext().getScan();
+ Filter filter = scan.getFilter();
+ ColumnExpression idExpression = new ColumnRef(plan.getTableRef(), plan.getTableRef().getTable().getColumn("ID").getPosition()).newColumnExpression();
+ Expression id = new RowKeyColumnExpression(idExpression,new RowKeyValueAccessor(plan.getTableRef().getTable().getPKColumns(),0));
+ Expression company = new KeyValueColumnExpression(plan.getTableRef().getTable().getColumn("COMPANY"));
+ // FilterList has no equals implementation
+ assertTrue(filter instanceof FilterList);
+ FilterList filterList = (FilterList)filter;
+ assertEquals(FilterList.Operator.MUST_PASS_ALL, filterList.getOperator());
+ assertEquals(
+ Arrays.asList(
+ new SkipScanFilter(
+ ImmutableList.of(Arrays.asList(
+ pointRange("i1"),
+ pointRange("i2"))),
+ SchemaUtil.VAR_BINARY_SCHEMA),
+ singleKVFilter(
+ or(constantComparison(CompareOp.EQUAL,id,"i1"),
+ and(constantComparison(CompareOp.EQUAL,id,"i2"),
+ constantComparison(CompareOp.EQUAL,company,"c3"))))),
+ filterList.getFilters());
+ }
+
+ @Test
+ public void testAndPKAndNotPK() throws SQLException {
+ String query = "select * from bugTable where ID = 'i2' and company = 'c3'";
+ PhoenixConnection pconn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)).unwrap(PhoenixConnection.class);
+ pconn.createStatement().execute("create table bugTable(ID varchar primary key,company varchar)");
+ PhoenixPreparedStatement pstmt = newPreparedStatement(pconn, query);
+ QueryPlan plan = pstmt.optimizeQuery();
+ Scan scan = plan.getContext().getScan();
+ Filter filter = scan.getFilter();
+ PColumn column = plan.getTableRef().getTable().getColumn("COMPANY");
+ assertEquals(
+ singleKVFilter(constantComparison(
+ CompareOp.EQUAL,
+ new KeyValueColumnExpression(column),
+ "c3")),
+ filter);
+ }
+
+ @Test
public void testSingleFixedFullPkSalted() throws SQLException {
PhoenixConnection pconn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)).unwrap(PhoenixConnection.class);
pconn.createStatement().execute("CREATE TABLE t (k bigint not null primary key, v varchar) SALT_BUCKETS=20");
[09/50] [abbrv] phoenix git commit: PHOENIX-2517 Bulk load tools
should support multiple input files
Posted by sa...@apache.org.
PHOENIX-2517 Bulk load tools should support multiple input files
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/f4257c6a
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/f4257c6a
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/f4257c6a
Branch: refs/heads/4.x-HBase-1.0
Commit: f4257c6a5c7ec8d32927b2d9987d5402740bfc47
Parents: 42506f4
Author: Nick Dimiduk <nd...@apache.org>
Authored: Fri Dec 11 12:41:49 2015 -0800
Committer: Nick Dimiduk <nd...@apache.org>
Committed: Tue Dec 15 12:36:36 2015 -0800
----------------------------------------------------------------------
.../phoenix/mapreduce/CsvBulkLoadToolIT.java | 41 ++++++++++++++++++++
.../phoenix/mapreduce/AbstractBulkLoadTool.java | 16 ++++----
.../mapreduce/MultiHfileOutputFormat.java | 2 +-
3 files changed, 50 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f4257c6a/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
index 4a440d6..7daacb4 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
@@ -36,6 +36,7 @@ import org.apache.phoenix.util.DateUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -166,6 +167,46 @@ public class CsvBulkLoadToolIT {
}
@Test
+ public void testMultipleInputFiles() throws Exception {
+
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE TABLE7 (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, T DATE) SPLIT ON (1,2)");
+
+ FileSystem fs = FileSystem.get(hbaseTestUtil.getConfiguration());
+ FSDataOutputStream outputStream = fs.create(new Path("/tmp/input1.csv"));
+ PrintWriter printWriter = new PrintWriter(outputStream);
+ printWriter.println("1,Name 1,1970/01/01");
+ printWriter.close();
+ outputStream = fs.create(new Path("/tmp/input2.csv"));
+ printWriter = new PrintWriter(outputStream);
+ printWriter.println("2,Name 2,1970/01/02");
+ printWriter.close();
+
+ CsvBulkLoadTool csvBulkLoadTool = new CsvBulkLoadTool();
+ csvBulkLoadTool.setConf(new Configuration(hbaseTestUtil.getConfiguration()));
+ csvBulkLoadTool.getConf().set(DATE_FORMAT_ATTRIB,"yyyy/MM/dd");
+ int exitCode = csvBulkLoadTool.run(new String[] {
+ "--input", "/tmp/input1.csv,/tmp/input2.csv",
+ "--table", "table7",
+ "--zookeeper", zkQuorum});
+ assertEquals(0, exitCode);
+
+ ResultSet rs = stmt.executeQuery("SELECT id, name, t FROM table7 ORDER BY id");
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals("Name 1", rs.getString(2));
+ assertEquals(DateUtil.parseDate("1970-01-01"), rs.getDate(3));
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+ assertEquals("Name 2", rs.getString(2));
+ assertEquals(DateUtil.parseDate("1970-01-02"), rs.getDate(3));
+ assertFalse(rs.next());
+
+ rs.close();
+ stmt.close();
+ }
+
+ @Test
public void testImportWithIndex() throws Exception {
Statement stmt = conn.createStatement();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f4257c6a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
index cf9ddef..1d2594d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
@@ -74,7 +74,7 @@ public abstract class AbstractBulkLoadTool extends Configured implements Tool {
protected static final Logger LOG = LoggerFactory.getLogger(AbstractBulkLoadTool.class);
static final Option ZK_QUORUM_OPT = new Option("z", "zookeeper", true, "Supply zookeeper connection details (optional)");
- static final Option INPUT_PATH_OPT = new Option("i", "input", true, "Input path (mandatory)");
+ static final Option INPUT_PATH_OPT = new Option("i", "input", true, "Input path(s) (comma-separated, mandatory)");
static final Option OUTPUT_PATH_OPT = new Option("o", "output", true, "Output path for temporary HFiles (optional)");
static final Option SCHEMA_NAME_OPT = new Option("s", "schema", true, "Phoenix schema name (optional)");
static final Option TABLE_NAME_OPT = new Option("t", "table", true, "Phoenix table name (mandatory)");
@@ -219,7 +219,7 @@ public abstract class AbstractBulkLoadTool extends Configured implements Tool {
conn.close();
}
- final Path inputPath = new Path(cmdLine.getOptionValue(INPUT_PATH_OPT.getOpt()));
+ final String inputPaths = cmdLine.getOptionValue(INPUT_PATH_OPT.getOpt());
final Path outputPath;
if (cmdLine.hasOption(OUTPUT_PATH_OPT.getOpt())) {
outputPath = new Path(cmdLine.getOptionValue(OUTPUT_PATH_OPT.getOpt()));
@@ -249,18 +249,18 @@ public abstract class AbstractBulkLoadTool extends Configured implements Tool {
tablesToBeLoaded.add(targetIndexRef);
}
- return submitJob(conf, tableName, inputPath, outputPath, tablesToBeLoaded);
+ return submitJob(conf, tableName, inputPaths, outputPath, tablesToBeLoaded);
}
/**
* Submits the jobs to the cluster.
* Loads the HFiles onto the respective tables.
*/
- public int submitJob(final Configuration conf, final String qualifiedTableName, final Path inputPath,
- final Path outputPath , List<TargetTableRef> tablesToBeLoaded) {
+ public int submitJob(final Configuration conf, final String qualifiedTableName,
+ final String inputPaths, final Path outputPath, List<TargetTableRef> tablesToBeLoaded) {
try {
Job job = new Job(conf, "Phoenix MapReduce import for " + qualifiedTableName);
- FileInputFormat.addInputPath(job, inputPath);
+ FileInputFormat.addInputPaths(job, inputPaths);
FileOutputFormat.setOutputPath(job, outputPath);
job.setInputFormatClass(TextInputFormat.class);
@@ -278,7 +278,7 @@ public abstract class AbstractBulkLoadTool extends Configured implements Tool {
// give subclasses their hook
setupJob(job);
- LOG.info("Running MapReduce import job from {} to {}", inputPath, outputPath);
+ LOG.info("Running MapReduce import job from {} to {}", inputPaths, outputPath);
boolean success = job.waitForCompletion(true);
if (success) {
@@ -292,7 +292,7 @@ public abstract class AbstractBulkLoadTool extends Configured implements Tool {
}
return 0;
} catch(Exception e) {
- LOG.error("Error {} occurred submitting BulkLoad ",e.getMessage());
+ LOG.error("Error occurred submitting BulkLoad ", e);
return -1;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f4257c6a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java
index 45a014d..525e79a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java
@@ -687,7 +687,7 @@ public class MultiHfileOutputFormat extends FileOutputFormat<TableRowkeyPair, Ce
conf.set(tableName, tableDefns);
TargetTableRef tbl = TargetTableRefFunctions.FROM_JSON.apply(tableDefns);
- LOG.error(" the table logical name is "+ tbl.getLogicalName());
+ LOG.info(" the table logical name is "+ tbl.getLogicalName());
}
}
[14/50] [abbrv] phoenix git commit: PHOENIX-2524 Fixes for pherf to
run with queryserver (elserj)
Posted by sa...@apache.org.
PHOENIX-2524 Fixes for pherf to run with queryserver (elserj)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/9dce7c11
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/9dce7c11
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/9dce7c11
Branch: refs/heads/4.x-HBase-1.0
Commit: 9dce7c11dfb96ea6ceb8eaf417e8f511b1fcbd9b
Parents: feec7c4
Author: Mujtaba <mu...@apache.org>
Authored: Fri Dec 18 15:39:40 2015 -0800
Committer: Mujtaba <mu...@apache.org>
Committed: Fri Dec 18 15:39:40 2015 -0800
----------------------------------------------------------------------
.../src/build/components/all-common-jars.xml | 8 +++
.../java/org/apache/phoenix/pherf/Pherf.java | 19 ++++++-
.../apache/phoenix/pherf/result/ResultUtil.java | 13 ++++-
.../phoenix/pherf/result/file/Header.java | 1 +
.../pherf/result/file/ResultFileDetails.java | 1 +
.../apache/phoenix/pherf/util/PhoenixUtil.java | 57 ++++++++++++++++----
.../phoenix/pherf/workload/WriteWorkload.java | 33 +++++++-----
7 files changed, 105 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9dce7c11/phoenix-assembly/src/build/components/all-common-jars.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/components/all-common-jars.xml b/phoenix-assembly/src/build/components/all-common-jars.xml
index bed9f25..960c3c9 100644
--- a/phoenix-assembly/src/build/components/all-common-jars.xml
+++ b/phoenix-assembly/src/build/components/all-common-jars.xml
@@ -104,5 +104,13 @@
</excludes>
<fileMode>0644</fileMode>
</fileSet>
+ <fileSet>
+ <directory>${project.basedir}/../phoenix-pherf/target/</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>phoenix-*.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
</fileSets>
</component>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9dce7c11/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
index eaf199a..b84183b 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
@@ -89,6 +89,8 @@ public class Pherf {
options.addOption("label", true, "Label a run. Result file name will be suffixed with specified label");
options.addOption("compare", true, "Specify labeled run(s) to compare");
options.addOption("useAverageCompareType", false, "Compare results with Average query time instead of default is Minimum query time.");
+ options.addOption("t", "thin", false, "Use the Phoenix Thin Driver");
+ options.addOption("s", "server", true, "The URL for the Phoenix QueryServer");
}
private final String zookeeper;
@@ -109,6 +111,8 @@ public class Pherf {
private final String label;
private final String compareResults;
private final CompareType compareType;
+ private final boolean thinDriver;
+ private final String queryServerUrl;
public Pherf(String[] args) throws Exception {
CommandLineParser parser = new PosixParser();
@@ -155,14 +159,27 @@ public class Pherf {
label = command.getOptionValue("label", null);
compareResults = command.getOptionValue("compare", null);
compareType = command.hasOption("useAverageCompareType") ? CompareType.AVERAGE : CompareType.MINIMUM;
+ thinDriver = command.hasOption("thin");
+ if (thinDriver) {
+ queryServerUrl = command.getOptionValue("server", "http://localhost:8765");
+ } else {
+ queryServerUrl = null;
+ }
if ((command.hasOption("h") || (args == null || args.length == 0)) && !command
.hasOption("listFiles")) {
hf.printHelp("Pherf", options);
System.exit(1);
}
- PhoenixUtil.setZookeeper(zookeeper);
PhoenixUtil.setRowCountOverride(rowCountOverride);
+ if (!thinDriver) {
+ logger.info("Using thick driver with ZooKeepers '{}'", zookeeper);
+ PhoenixUtil.setZookeeper(zookeeper);
+ } else {
+ logger.info("Using thin driver with PQS '{}'", queryServerUrl);
+ // Enables the thin-driver and sets the PQS URL
+ PhoenixUtil.useThinDriver(queryServerUrl);
+ }
ResultUtil.setFileSuffix(label);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9dce7c11/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultUtil.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultUtil.java
index 92eb80a..0c2a7b8 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultUtil.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultUtil.java
@@ -84,7 +84,12 @@ public class ResultUtil {
ensureBaseResultDirExists();
CSVResultHandler writer = null;
- ResultFileDetails resultFileDetails = ResultFileDetails.CSV_AGGREGATE_DATA_LOAD;
+ ResultFileDetails resultFileDetails;
+ if (PhoenixUtil.isThinDriver()) {
+ resultFileDetails = ResultFileDetails.CSV_THIN_AGGREGATE_DATA_LOAD;
+ } else {
+ resultFileDetails = ResultFileDetails.CSV_AGGREGATE_DATA_LOAD;
+ }
try {
writer = new CSVFileResultHandler();
writer.setResultFileDetails(resultFileDetails);
@@ -92,7 +97,11 @@ public class ResultUtil {
for (TableLoadTime loadTime : dataLoadTime.getTableLoadTime()) {
List<ResultValue> rowValues = new ArrayList<>();
- rowValues.add(new ResultValue(PhoenixUtil.getZookeeper()));
+ if (PhoenixUtil.isThinDriver()) {
+ rowValues.add(new ResultValue(PhoenixUtil.getQueryServerUrl()));
+ } else {
+ rowValues.add(new ResultValue(PhoenixUtil.getZookeeper()));
+ }
rowValues.addAll(loadTime.getCsvRepresentation(this));
Result
result =
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9dce7c11/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/file/Header.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/file/Header.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/file/Header.java
index 066fa7a..7d09f68 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/file/Header.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/file/Header.java
@@ -28,6 +28,7 @@ public enum Header {
DETAILED_PERFORMANCE(DETAILED_BASE + ",RESULT_ROW_COUNT,RUN_TIME_MS"),
DETAILED_FUNCTIONAL(DETAILED_BASE + ",DIFF_STATUS,EXPLAIN_PLAN"),
AGGREGATE_DATA_LOAD("ZK,TABLE_NAME,ROW_COUNT,LOAD_DURATION_IN_MS"),
+ THIN_AGGREGATE_DATA_LOAD("QUERYSERVER,TABLE_NAME,ROW_COUNT,LOAD_DURATION_IN_MS"),
MONITOR("STAT_NAME,STAT_VALUE,TIME_STAMP");
private String header;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9dce7c11/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/file/ResultFileDetails.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/file/ResultFileDetails.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/file/ResultFileDetails.java
index a85f830..51aa407 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/file/ResultFileDetails.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/file/ResultFileDetails.java
@@ -24,6 +24,7 @@ public enum ResultFileDetails {
CSV_DETAILED_PERFORMANCE(Header.DETAILED_PERFORMANCE, Extension.DETAILED_CSV),
CSV_DETAILED_FUNCTIONAL(Header.DETAILED_FUNCTIONAL, Extension.DETAILED_CSV),
CSV_AGGREGATE_DATA_LOAD(Header.AGGREGATE_DATA_LOAD, Extension.CSV),
+ CSV_THIN_AGGREGATE_DATA_LOAD(Header.THIN_AGGREGATE_DATA_LOAD, Extension.CSV),
CSV_MONITOR(Header.MONITOR, Extension.CSV),
XML(Header.EMPTY, Extension.XML),
IMAGE(Header.EMPTY, Extension.VISUALIZATION);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9dce7c11/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
index 064c604..b778833 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
@@ -28,6 +28,7 @@ import java.sql.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Objects;
import java.util.Properties;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
@@ -39,6 +40,8 @@ public class PhoenixUtil {
private static int rowCountOverride = 0;
private boolean testEnabled;
private static PhoenixUtil instance;
+ private static boolean useThinDriver;
+ private static String queryServerUrl;
private PhoenixUtil() {
this(false);
@@ -57,6 +60,19 @@ public class PhoenixUtil {
return instance;
}
+ public static void useThinDriver(String queryServerUrl) {
+ PhoenixUtil.useThinDriver = true;
+ PhoenixUtil.queryServerUrl = Objects.requireNonNull(queryServerUrl);
+ }
+
+ public static String getQueryServerUrl() {
+ return PhoenixUtil.queryServerUrl;
+ }
+
+ public static boolean isThinDriver() {
+ return PhoenixUtil.useThinDriver;
+ }
+
public Connection getConnection() throws Exception {
return getConnection(null);
}
@@ -66,17 +82,31 @@ public class PhoenixUtil {
}
private Connection getConnection(String tenantId, boolean testEnabled) throws Exception {
- if (null == zookeeper) {
- throw new IllegalArgumentException(
- "Zookeeper must be set before initializing connection!");
- }
- Properties props = new Properties();
- if (null != tenantId) {
- props.setProperty("TenantId", tenantId);
- logger.debug("\nSetting tenantId to " + tenantId);
+ if (useThinDriver) {
+ if (null == queryServerUrl) {
+ throw new IllegalArgumentException("QueryServer URL must be set before" +
+ " initializing connection");
+ }
+ Properties props = new Properties();
+ if (null != tenantId) {
+ props.setProperty("TenantId", tenantId);
+ logger.debug("\nSetting tenantId to " + tenantId);
+ }
+ String url = "jdbc:phoenix:thin:url=" + queryServerUrl + ";serialization=PROTOBUF";
+ return DriverManager.getConnection(url, props);
+ } else {
+ if (null == zookeeper) {
+ throw new IllegalArgumentException(
+ "Zookeeper must be set before initializing connection!");
+ }
+ Properties props = new Properties();
+ if (null != tenantId) {
+ props.setProperty("TenantId", tenantId);
+ logger.debug("\nSetting tenantId to " + tenantId);
+ }
+ String url = "jdbc:phoenix:" + zookeeper + (testEnabled ? ";test=true" : "");
+ return DriverManager.getConnection(url, props);
}
- String url = "jdbc:phoenix:" + zookeeper + (testEnabled ? ";test=true" : "");
- return DriverManager.getConnection(url, props);
}
public boolean executeStatement(String sql, Scenario scenario) throws Exception {
@@ -278,7 +308,12 @@ public class PhoenixUtil {
public static void setZookeeper(String zookeeper) {
logger.info("Setting zookeeper: " + zookeeper);
- PhoenixUtil.zookeeper = zookeeper;
+ useThickDriver(zookeeper);
+ }
+
+ public static void useThickDriver(String zookeeper) {
+ PhoenixUtil.useThinDriver = false;
+ PhoenixUtil.zookeeper = Objects.requireNonNull(zookeeper);
}
public static int getRowCountOverride() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9dce7c11/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
index 7c73e22..a35e6e8 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
@@ -168,7 +168,7 @@ public class WriteWorkload implements Workload {
// Execute any Scenario DDL before running workload
pUtil.executeScenarioDdl(scenario);
- List<Future> writeBatches = getBatches(dataLoadThreadTime, scenario);
+ List<Future<Info>> writeBatches = getBatches(dataLoadThreadTime, scenario);
waitForBatches(dataLoadTimeSummary, scenario, start, writeBatches);
@@ -182,12 +182,12 @@ public class WriteWorkload implements Workload {
}
}
- private List<Future> getBatches(DataLoadThreadTime dataLoadThreadTime, Scenario scenario)
+ private List<Future<Info>> getBatches(DataLoadThreadTime dataLoadThreadTime, Scenario scenario)
throws Exception {
RowCalculator
rowCalculator =
new RowCalculator(getThreadPoolSize(), scenario.getRowCount());
- List<Future> writeBatches = new ArrayList<>();
+ List<Future<Info>> writeBatches = new ArrayList<>();
for (int i = 0; i < getThreadPoolSize(); i++) {
List<Column>
@@ -212,7 +212,7 @@ public class WriteWorkload implements Workload {
}
private void waitForBatches(DataLoadTimeSummary dataLoadTimeSummary, Scenario scenario,
- long start, List<Future> writeBatches)
+ long start, List<Future<Info>> writeBatches)
throws InterruptedException, java.util.concurrent.ExecutionException {
int sumRows = 0, sumDuration = 0;
// Wait for all the batch threads to complete
@@ -223,10 +223,12 @@ public class WriteWorkload implements Workload {
logger.info("Executor (" + this.hashCode() + ") writes complete with row count ("
+ writeInfo.getRowCount() + ") in Ms (" + writeInfo.getDuration() + ")");
}
- logger.info("Writes completed with total row count (" + sumRows + ") with total time of("
- + sumDuration + ") Ms");
+ long testDuration = System.currentTimeMillis() - start;
+ logger.info("Writes completed with total row count (" + sumRows
+ + ") with total elapsed time of (" + testDuration
+ + ") ms and total CPU execution time of (" + sumDuration + ") ms");
dataLoadTimeSummary
- .add(scenario.getTableName(), sumRows, (int) (System.currentTimeMillis() - start));
+ .add(scenario.getTableName(), sumRows, (int) testDuration);
}
public Future<Info> upsertData(final Scenario scenario, final List<Column> columns,
@@ -235,9 +237,10 @@ public class WriteWorkload implements Workload {
Future<Info> future = pool.submit(new Callable<Info>() {
@Override public Info call() throws Exception {
int rowsCreated = 0;
- long start = 0, duration, totalDuration;
+ long start = 0, last = 0, duration, totalDuration;
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Connection connection = null;
+ PreparedStatement stmt = null;
try {
connection = pUtil.getConnection(scenario.getTenantId());
long logStartTime = System.currentTimeMillis();
@@ -247,17 +250,16 @@ public class WriteWorkload implements Workload {
Long.MAX_VALUE :
WriteWorkload.this.writeParams.getExecutionDurationInMs();
+ last = start = System.currentTimeMillis();
+ String sql = buildSql(columns, tableName);
+ stmt = connection.prepareStatement(sql);
for (long i = rowCount; (i > 0) && ((System.currentTimeMillis() - logStartTime)
< maxDuration); i--) {
- String sql = buildSql(columns, tableName);
- PreparedStatement stmt = connection.prepareStatement(sql);
stmt = buildStatement(scenario, columns, stmt, simpleDateFormat);
- start = System.currentTimeMillis();
rowsCreated += stmt.executeUpdate();
- stmt.close();
if ((i % getBatchSize()) == 0) {
connection.commit();
- duration = System.currentTimeMillis() - start;
+ duration = System.currentTimeMillis() - last;
logger.info("Writer (" + Thread.currentThread().getName()
+ ") committed Batch. Total " + getBatchSize()
+ " rows for this thread (" + this.hashCode() + ") in ("
@@ -272,9 +274,14 @@ public class WriteWorkload implements Workload {
// Pause for throttling if configured to do so
Thread.sleep(threadSleepDuration);
+ // Re-compute the start time for the next batch
+ last = System.currentTimeMillis();
}
}
} finally {
+ if (stmt != null) {
+ stmt.close();
+ }
if (connection != null) {
try {
connection.commit();
[48/50] [abbrv] phoenix git commit: PHOENIX-2357 Prevent queries from
failing because of scanners running into lease expiration
Posted by sa...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/test/java/org/apache/phoenix/query/ScannerLeaseRenewalTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/ScannerLeaseRenewalTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/ScannerLeaseRenewalTest.java
new file mode 100644
index 0000000..7d8904d
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/ScannerLeaseRenewalTest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.query;
+
+import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.CLOSED;
+import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.NOT_RENEWED;
+import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.RENEWED;
+import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.THRESHOLD_NOT_REACHED;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.lang.ref.WeakReference;
+import java.sql.DriverManager;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.apache.phoenix.iterate.RenewLeaseOnlyTableIterator;
+import org.apache.phoenix.iterate.TableResultIterator;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.ConnectionQueryServicesImpl.RenewLeaseTask;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.junit.Test;
+
+public class ScannerLeaseRenewalTest extends BaseConnectionlessQueryTest {
+
+ @Test
+ public void testRenewLeaseTaskBehavior() throws Exception {
+ // add connection to the queue
+ PhoenixConnection pconn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)).unwrap(PhoenixConnection.class);
+ LinkedBlockingQueue<WeakReference<PhoenixConnection>> connectionsQueue = new LinkedBlockingQueue<>();
+ connectionsQueue.add(new WeakReference<PhoenixConnection>(pconn));
+
+ // create a scanner and add it to the queue
+ int numLeaseRenewals = 4;
+ int skipRenewLeaseCount = 2;
+ RenewLeaseOnlyTableIterator itr = new RenewLeaseOnlyTableIterator(numLeaseRenewals, skipRenewLeaseCount, -1);
+ LinkedBlockingQueue<WeakReference<TableResultIterator>> scannerQueue = pconn.getScanners();
+ scannerQueue.add(new WeakReference<TableResultIterator>(itr));
+
+ RenewLeaseTask task = new RenewLeaseTask(connectionsQueue);
+ assertTrue(connectionsQueue.size() == 1);
+ assertTrue(scannerQueue.size() == 1);
+
+ task.run();
+ assertTrue(connectionsQueue.size() == 1);
+ assertTrue(scannerQueue.size() == 1); // lease renewed
+ assertEquals(RENEWED, itr.getLastRenewLeaseStatus());
+
+ task.run();
+ assertTrue(scannerQueue.size() == 1);
+ assertTrue(connectionsQueue.size() == 1); // renew lease skipped but scanner still in the queue
+ assertEquals(THRESHOLD_NOT_REACHED, itr.getLastRenewLeaseStatus());
+
+ task.run();
+ assertTrue(scannerQueue.size() == 1);
+ assertTrue(connectionsQueue.size() == 1);
+ assertEquals(RENEWED, itr.getLastRenewLeaseStatus()); // lease renewed
+
+ task.run();
+ assertTrue(scannerQueue.size() == 1);
+ assertTrue(connectionsQueue.size() == 1);
+ assertEquals(RENEWED, itr.getLastRenewLeaseStatus()); // lease renewed
+
+ task.run();
+ assertTrue(scannerQueue.size() == 0);
+ assertTrue(connectionsQueue.size() == 1);
+ assertEquals(CLOSED, itr.getLastRenewLeaseStatus()); // scanner closed and removed from the queue
+
+ pconn.close();
+ task.run();
+ assertTrue(scannerQueue.size() == 0);
+ assertTrue("Closing the connection should have removed it from the queue", connectionsQueue.size() == 0);
+ }
+
+ @Test
+ public void testRenewLeaseTaskBehaviorOnError() throws Exception {
+ // add connection to the queue
+ PhoenixConnection pconn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)).unwrap(PhoenixConnection.class);
+ LinkedBlockingQueue<WeakReference<PhoenixConnection>> connectionsQueue = new LinkedBlockingQueue<>();
+ connectionsQueue.add(new WeakReference<PhoenixConnection>(pconn));
+
+ // create a scanner and add it to the queue
+ int numLeaseRenewals = 4;
+ int thresholdNotReachedCount = 2;
+ int leaseNotRenewedCount = 3;
+ RenewLeaseOnlyTableIterator itr = new RenewLeaseOnlyTableIterator(numLeaseRenewals, thresholdNotReachedCount, leaseNotRenewedCount);
+ LinkedBlockingQueue<WeakReference<TableResultIterator>> scannerQueue = pconn.getScanners();
+ scannerQueue.add(new WeakReference<TableResultIterator>(itr));
+
+ RenewLeaseTask task = new RenewLeaseTask(connectionsQueue);
+ assertTrue(connectionsQueue.size() == 1);
+ assertTrue(scannerQueue.size() == 1);
+
+ task.run();
+ assertTrue(connectionsQueue.size() == 1);
+ assertTrue(scannerQueue.size() == 1); // lease renewed
+ assertEquals(RENEWED, itr.getLastRenewLeaseStatus());
+
+ task.run();
+ assertTrue(scannerQueue.size() == 1);
+ assertTrue(connectionsQueue.size() == 1); // renew lease skipped but scanner still in the queue
+ assertEquals(THRESHOLD_NOT_REACHED, itr.getLastRenewLeaseStatus());
+
+ task.run();
+ assertTrue(scannerQueue.size() == 0);
+ assertTrue(connectionsQueue.size() == 1);
+ // Lease not renewed due to error or some other reason.
+ // In this case we don't call renew lease on the scanner anymore.
+ assertEquals(NOT_RENEWED, itr.getLastRenewLeaseStatus());
+
+ pconn.close();
+ task.run();
+ assertTrue(scannerQueue.size() == 0);
+ assertTrue("Closing the connection should have removed it from the queue", connectionsQueue.size() == 0);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
index 9597b9c..1e06379 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
@@ -41,6 +41,7 @@ public class MetaDataUtilTest {
assertTrue(VersionUtil.encodeVersion("0.94.6")>VersionUtil.encodeVersion("0.94.5-mapR"));
assertTrue(VersionUtil.encodeVersion("0.94.6")>VersionUtil.encodeVersion("0.94.5"));
assertTrue(VersionUtil.encodeVersion("0.94.1-mapR")>VersionUtil.encodeVersion("0.94"));
+ assertTrue(VersionUtil.encodeVersion("1", "1", "3")>VersionUtil.encodeVersion("1", "1", "1"));
}
@Test
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5fb51ca5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0b65a66..eebba33 100644
--- a/pom.xml
+++ b/pom.xml
@@ -80,7 +80,7 @@
<top.dir>${project.basedir}</top.dir>
<!-- Hadoop Versions -->
- <hbase.version>1.0.1</hbase.version>
+ <hbase.version>1.0.2</hbase.version>
<hadoop-two.version>2.5.1</hadoop-two.version>
<!-- Dependency versions -->
[11/50] [abbrv] phoenix git commit: PHOENIX-2531 Explicitly list
java.sql.Driver file to fix thin-client jar
Posted by sa...@apache.org.
PHOENIX-2531 Explicitly list java.sql.Driver file to fix thin-client jar
The file with the same name was being included in the final artifact
from including the Avatica jar in this jar. If we specify the file
explicitly, we can be sure that it will be the file contained in
the jar.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/356b73f1
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/356b73f1
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/356b73f1
Branch: refs/heads/4.x-HBase-1.0
Commit: 356b73f1a9e3224ee97cae7d13a3918b6c88dfb8
Parents: 9a71114
Author: Josh Elser <el...@apache.org>
Authored: Tue Dec 15 17:01:57 2015 -0500
Committer: Nick Dimiduk <nd...@apache.org>
Committed: Tue Dec 15 15:51:14 2015 -0800
----------------------------------------------------------------------
phoenix-server-client/src/build/thin-client.xml | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/356b73f1/phoenix-server-client/src/build/thin-client.xml
----------------------------------------------------------------------
diff --git a/phoenix-server-client/src/build/thin-client.xml b/phoenix-server-client/src/build/thin-client.xml
index 5865395..0e7ca52 100644
--- a/phoenix-server-client/src/build/thin-client.xml
+++ b/phoenix-server-client/src/build/thin-client.xml
@@ -38,4 +38,12 @@
<scope>runtime</scope>
</dependencySet>
</dependencySets>
+ <files>
+ <file>
+ <!-- Specify this file directly to make sure we don't get the file
+ with the same path/name from Avatica. -->
+ <source>src/main/resources/META-INF/services/java.sql.Driver</source>
+ <outputDirectory>/META-INF/services/</outputDirectory>
+ </file>
+ </files>
</assembly>
[21/50] [abbrv] phoenix git commit: Merge branch '4.x-HBase-1.0' of
https://git-wip-us.apache.org/repos/asf/phoenix into 4.x-HBase-1.0
Posted by sa...@apache.org.
Merge branch '4.x-HBase-1.0' of https://git-wip-us.apache.org/repos/asf/phoenix into 4.x-HBase-1.0
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7ca36987
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7ca36987
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7ca36987
Branch: refs/heads/4.x-HBase-1.0
Commit: 7ca36987af5f30587fd9b3ed8768d95fc1a6e604
Parents: 08aa07f 2f8d71d
Author: Samarth <sa...@apache.org>
Authored: Tue Dec 22 14:01:28 2015 -0800
Committer: Samarth <sa...@apache.org>
Committed: Tue Dec 22 14:01:28 2015 -0800
----------------------------------------------------------------------
.../mapreduce/PhoenixRecordWritable.java | 192 +++++++++++++++++++
1 file changed, 192 insertions(+)
----------------------------------------------------------------------
[32/50] [abbrv] phoenix git commit: PHOENIX-2550 Increase JDBC
major/minor version for 4.7 release
Posted by sa...@apache.org.
PHOENIX-2550 Increase JDBC major/minor version for 4.7 release
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/9e66b35f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/9e66b35f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/9e66b35f
Branch: refs/heads/4.x-HBase-1.0
Commit: 9e66b35ffda579cbf55e792d404833d701c3b91c
Parents: 708a800
Author: James Taylor <jt...@salesforce.com>
Authored: Wed Dec 30 09:32:53 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Dec 30 18:14:26 2015 -0800
----------------------------------------------------------------------
.../main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9e66b35f/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index ba06828..bf4a192 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -55,7 +55,7 @@ import com.google.protobuf.ByteString;
*/
public abstract class MetaDataProtocol extends MetaDataService {
public static final int PHOENIX_MAJOR_VERSION = 4;
- public static final int PHOENIX_MINOR_VERSION = 6;
+ public static final int PHOENIX_MINOR_VERSION = 7;
public static final int PHOENIX_PATCH_NUMBER = 0;
public static final int PHOENIX_VERSION =
VersionUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER);
[41/50] [abbrv] phoenix git commit: PHOENIX-2562 Change commit call
to send for batched autocommit deletes
Posted by sa...@apache.org.
PHOENIX-2562 Change commit call to send for batched autocommit deletes
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c027e0b0
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c027e0b0
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c027e0b0
Branch: refs/heads/4.x-HBase-1.0
Commit: c027e0b0ba61fca57e4e0f66647e61593be0586e
Parents: 423bd13
Author: James Taylor <jt...@salesforce.com>
Authored: Mon Jan 4 12:11:00 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Jan 4 12:11:59 2016 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/phoenix/compile/DeleteCompiler.java | 2 +-
.../src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java | 1 -
2 files changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c027e0b0/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
index 12aa7c5..ac9f7d1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
@@ -172,7 +172,7 @@ public class DeleteCompiler {
MutationState indexState = new MutationState(indexTableRef, indexMutations, 0, maxSize, connection);
connection.getMutationState().join(indexState);
}
- connection.commit();
+ connection.getMutationState().send();
mutations.clear();
if (indexMutations != null) {
indexMutations.clear();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c027e0b0/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
index c39df2c..00b6637 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
@@ -192,7 +192,6 @@ public class PhoenixMetricsSink implements MetricsSink {
public void flush() {
try {
this.conn.commit();
- this.conn.rollback();
} catch (SQLException e) {
LOG.error("Failed to commit changes to table", e);
}
[10/50] [abbrv] phoenix git commit: PHOENIX-2510 ReserveNSequence
opens new connection per invocation (Siddhi Mehta)
Posted by sa...@apache.org.
PHOENIX-2510 ReserveNSequence opens new connection per invocation (Siddhi Mehta)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/9a71114d
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/9a71114d
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/9a71114d
Branch: refs/heads/4.x-HBase-1.0
Commit: 9a71114d8f16c597ae3407e9f24c414d24165809
Parents: f4257c6
Author: James Taylor <ja...@apache.org>
Authored: Tue Dec 15 14:06:14 2015 -0800
Committer: James Taylor <ja...@apache.org>
Committed: Tue Dec 15 14:10:15 2015 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/pig/udf/ReserveNSequenceTestIT.java | 2 ++
.../java/org/apache/phoenix/pig/udf/ReserveNSequence.java | 9 ---------
2 files changed, 2 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9a71114d/phoenix-pig/src/it/java/org/apache/phoenix/pig/udf/ReserveNSequenceTestIT.java
----------------------------------------------------------------------
diff --git a/phoenix-pig/src/it/java/org/apache/phoenix/pig/udf/ReserveNSequenceTestIT.java b/phoenix-pig/src/it/java/org/apache/phoenix/pig/udf/ReserveNSequenceTestIT.java
index 30ce132..98f46f0 100644
--- a/phoenix-pig/src/it/java/org/apache/phoenix/pig/udf/ReserveNSequenceTestIT.java
+++ b/phoenix-pig/src/it/java/org/apache/phoenix/pig/udf/ReserveNSequenceTestIT.java
@@ -193,6 +193,8 @@ public class ReserveNSequenceTestIT extends BasePigIT {
ReserveNSequence udf = new ReserveNSequence(zkQuorum, tenantId);
result = udf.exec(tuple);
validateReservedSequence(conn, props.getCurrentValue(), props.getNumToReserve(), result);
+ // Calling this to cleanup for the udf. To close the connection
+ udf.finish();
} catch (Exception e) {
if (props.isExceptionExpected()) {
assertEquals(props.getExceptionClass(), e.getClass());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9a71114d/phoenix-pig/src/main/java/org/apache/phoenix/pig/udf/ReserveNSequence.java
----------------------------------------------------------------------
diff --git a/phoenix-pig/src/main/java/org/apache/phoenix/pig/udf/ReserveNSequence.java b/phoenix-pig/src/main/java/org/apache/phoenix/pig/udf/ReserveNSequence.java
index 6187d5e..eaf4e91 100644
--- a/phoenix-pig/src/main/java/org/apache/phoenix/pig/udf/ReserveNSequence.java
+++ b/phoenix-pig/src/main/java/org/apache/phoenix/pig/udf/ReserveNSequence.java
@@ -77,7 +77,6 @@ public class ReserveNSequence extends EvalFunc<Long> {
}
ResultSet rs = null;
try {
- connection = ConnectionUtil.getOutputConnection(configuration);
String sql = getNextNSequenceSelectStatement(Long.valueOf(numToReserve), sequenceName);
rs = connection.createStatement().executeQuery(sql);
Preconditions.checkArgument(rs.next());
@@ -87,14 +86,6 @@ public class ReserveNSequence extends EvalFunc<Long> {
return startIndex;
} catch (SQLException e) {
throw new IOException("Caught exception while processing row." + e.getMessage(), e);
- } finally {
- try {
- if (connection != null) {
- connection.close();
- }
- } catch (SQLException e) {
- throw new IOException("Caught exception while closing connection", e);
- }
}
}
[04/50] [abbrv] phoenix git commit: PHOENIX-2332 Fix timing out of
MutableIndexFailureIT.testWriteFailureDisablesIndex()
Posted by sa...@apache.org.
PHOENIX-2332 Fix timing out of MutableIndexFailureIT.testWriteFailureDisablesIndex()
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8726efd1
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8726efd1
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8726efd1
Branch: refs/heads/4.x-HBase-1.0
Commit: 8726efd13378e622659982b688f6cc76a2c20aca
Parents: b06a20c
Author: James Taylor <jt...@salesforce.com>
Authored: Mon Dec 14 11:45:17 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Dec 14 11:51:30 2015 -0800
----------------------------------------------------------------------
.../end2end/index/MutableIndexFailureIT.java | 264 ++++++++++---------
1 file changed, 133 insertions(+), 131 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8726efd1/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index eac15b9..4a4d058 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -61,6 +61,7 @@ import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.StringUtil;
import org.apache.phoenix.util.TestUtil;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
@@ -99,7 +100,7 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT {
this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
this.fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
}
-
+
@BeforeClass
public static void doSetup() throws Exception {
Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(10);
@@ -111,7 +112,7 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT {
NUM_SLAVES_BASE = 4;
setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator()));
}
-
+
@Parameters(name = "transactional = {0}")
public static Collection<Boolean[]> data() {
return Arrays.asList(new Boolean[][] { { false }, { true } });
@@ -121,12 +122,12 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT {
public void testWriteFailureDisablesLocalIndex() throws Exception {
helpTestWriteFailureDisablesIndex(true);
}
-
+
@Test
public void testWriteFailureDisablesIndex() throws Exception {
helpTestWriteFailureDisablesIndex(false);
}
-
+
public void helpTestWriteFailureDisablesIndex(boolean localIndex) throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = driver.connect(url, props)) {
@@ -138,21 +139,21 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT {
query = "SELECT * FROM " + fullTableName;
rs = conn.createStatement().executeQuery(query);
assertFalse(rs.next());
-
+
if(localIndex) {
conn.createStatement().execute(
- "CREATE LOCAL INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE (v2)");
+ "CREATE LOCAL INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE (v2)");
conn.createStatement().execute(
- "CREATE LOCAL INDEX " + indexName+ "_2" + " ON " + fullTableName + " (v2) INCLUDE (v1)");
+ "CREATE LOCAL INDEX " + indexName+ "_2" + " ON " + fullTableName + " (v2) INCLUDE (v1)");
} else {
conn.createStatement().execute(
- "CREATE INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE (v2)");
+ "CREATE INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE (v2)");
}
-
+
query = "SELECT * FROM " + fullIndexName;
rs = conn.createStatement().executeQuery(query);
assertFalse(rs.next());
-
+
// Verify the metadata for index is correct.
rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
new String[] { PTableType.INDEX.toString() });
@@ -160,24 +161,24 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT {
assertEquals(indexName, rs.getString(3));
assertEquals(PIndexState.ACTIVE.toString(), rs.getString("INDEX_STATE"));
assertFalse(rs.next());
-
+
PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
stmt.setString(1, "a");
stmt.setString(2, "x");
stmt.setString(3, "1");
stmt.execute();
conn.commit();
-
+
TableName indexTable =
TableName.valueOf(localIndex ? MetaDataUtil
.getLocalIndexTableName(fullTableName) : fullIndexName);
HBaseAdmin admin = this.getUtility().getHBaseAdmin();
HTableDescriptor indexTableDesc = admin.getTableDescriptor(indexTable);
try{
- admin.disableTable(indexTable);
- admin.deleteTable(indexTable);
+ admin.disableTable(indexTable);
+ admin.deleteTable(indexTable);
} catch (TableNotFoundException ignore) {}
-
+
stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
stmt.setString(1, "a2");
stmt.setString(2, "x2");
@@ -194,7 +195,7 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT {
else {
conn.commit();
}
-
+
// Verify the metadata for index is correct.
rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
new String[] { PTableType.INDEX.toString() });
@@ -206,13 +207,13 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT {
assertFalse(rs.next());
if(localIndex) {
rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName + "_2",
- new String[] { PTableType.INDEX.toString() });
+ new String[] { PTableType.INDEX.toString() });
assertTrue(rs.next());
assertEquals(indexName + "_2", rs.getString(3));
assertEquals(indexState.toString(), rs.getString("INDEX_STATE"));
assertFalse(rs.next());
}
-
+
// if the table is transactional the write to the index table will fail because the
// index has not been disabled
if (!transactional) {
@@ -224,7 +225,7 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT {
stmt.execute();
conn.commit();
}
-
+
if (transactional) {
// if the table was transactional there should be 1 row (written before the index
// was disabled)
@@ -254,27 +255,27 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT {
assertEquals("3", rs.getString(1));
assertFalse(rs.next());
}
-
+
// recreate index table
admin.createTable(indexTableDesc);
do {
- Thread.sleep(15 * 1000); // sleep 15 secs
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
- new String[] { PTableType.INDEX.toString() });
- assertTrue(rs.next());
- if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
- break;
- }
- if(localIndex) {
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName + "_2",
- new String[] { PTableType.INDEX.toString() });
- assertTrue(rs.next());
- if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
- break;
- }
- }
+ Thread.sleep(15 * 1000); // sleep 15 secs
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
+ new String[] { PTableType.INDEX.toString() });
+ assertTrue(rs.next());
+ if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
+ break;
+ }
+ if(localIndex) {
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName + "_2",
+ new String[] { PTableType.INDEX.toString() });
+ assertTrue(rs.next());
+ if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
+ break;
+ }
+ }
} while(true);
-
+
// Verify UPSERT on data table still work after index table is recreated
stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
stmt.setString(1, "a4");
@@ -293,107 +294,108 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT {
// from where we failed and the oldest
// index row has been deleted when we dropped the index table during test
assertEquals(transactional ? 1 : 3, rs.getInt(1));
- }
}
-
- @Test
- public void testWriteFailureWithRegionServerDown() throws Exception {
- String query;
- ResultSet rs;
-
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- try (Connection conn = driver.connect(url, props);) {
- conn.setAutoCommit(false);
- conn.createStatement().execute(
- "CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) "+tableDDLOptions);
- query = "SELECT * FROM " + fullTableName;
- rs = conn.createStatement().executeQuery(query);
- assertFalse(rs.next());
-
- conn.createStatement().execute(
- "CREATE INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE (v2)");
- query = "SELECT * FROM " + fullIndexName;
- rs = conn.createStatement().executeQuery(query);
- assertFalse(rs.next());
-
- // Verify the metadata for index is correct.
+ }
+
+ @Ignore("See PHOENIX-2332")
+ @Test
+ public void testWriteFailureWithRegionServerDown() throws Exception {
+ String query;
+ ResultSet rs;
+
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = driver.connect(url, props);) {
+ conn.setAutoCommit(false);
+ conn.createStatement().execute(
+ "CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) "+tableDDLOptions);
+ query = "SELECT * FROM " + fullTableName;
+ rs = conn.createStatement().executeQuery(query);
+ assertFalse(rs.next());
+
+ conn.createStatement().execute(
+ "CREATE INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE (v2)");
+ query = "SELECT * FROM " + fullIndexName;
+ rs = conn.createStatement().executeQuery(query);
+ assertFalse(rs.next());
+
+ // Verify the metadata for index is correct.
+ rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
+ new String[] { PTableType.INDEX.toString() });
+ assertTrue(rs.next());
+ assertEquals(indexName, rs.getString(3));
+ assertEquals(PIndexState.ACTIVE.toString(), rs.getString("INDEX_STATE"));
+ assertFalse(rs.next());
+
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
+ stmt.setString(1, "a");
+ stmt.setString(2, "x");
+ stmt.setString(3, "1");
+ stmt.execute();
+ conn.commit();
+
+ // find a RS which doesn't has CATALOG table
+ TableName catalogTable = TableName.valueOf("SYSTEM.CATALOG");
+ TableName indexTable = TableName.valueOf(fullIndexName);
+ final HBaseCluster cluster = this.getUtility().getHBaseCluster();
+ Collection<ServerName> rss = cluster.getClusterStatus().getServers();
+ HBaseAdmin admin = this.getUtility().getHBaseAdmin();
+ List<HRegionInfo> regions = admin.getTableRegions(catalogTable);
+ ServerName catalogRS = cluster.getServerHoldingRegion(regions.get(0).getTable(),
+ regions.get(0).getRegionName());
+ ServerName metaRS = cluster.getServerHoldingMeta();
+ ServerName rsToBeKilled = null;
+
+ // find first RS isn't holding META or CATALOG table
+ for(ServerName curRS : rss) {
+ if(!curRS.equals(catalogRS) && !metaRS.equals(curRS)) {
+ rsToBeKilled = curRS;
+ break;
+ }
+ }
+ assertTrue(rsToBeKilled != null);
+
+ regions = admin.getTableRegions(indexTable);
+ final HRegionInfo indexRegion = regions.get(0);
+ final ServerName dstRS = rsToBeKilled;
+ admin.move(indexRegion.getEncodedNameAsBytes(), Bytes.toBytes(rsToBeKilled.getServerName()));
+ this.getUtility().waitFor(30000, 200, new Waiter.Predicate<Exception>() {
+ @Override
+ public boolean evaluate() throws Exception {
+ ServerName sn = cluster.getServerHoldingRegion(indexRegion.getTable(),
+ indexRegion.getRegionName());
+ return (sn != null && sn.equals(dstRS));
+ }
+ });
+
+ // use timer sending updates in every 10ms
+ this.scheduleTimer = new Timer(true);
+ this.scheduleTimer.schedule(new SendingUpdatesScheduleTask(conn, fullTableName), 0, 10);
+ // let timer sending some updates
+ Thread.sleep(100);
+
+ // kill RS hosting index table
+ this.getUtility().getHBaseCluster().killRegionServer(rsToBeKilled);
+
+ // wait for index table completes recovery
+ this.getUtility().waitUntilAllRegionsAssigned(indexTable);
+
+ // Verify the metadata for index is correct.
+ do {
+ Thread.sleep(15 * 1000); // sleep 15 secs
rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
new String[] { PTableType.INDEX.toString() });
assertTrue(rs.next());
- assertEquals(indexName, rs.getString(3));
- assertEquals(PIndexState.ACTIVE.toString(), rs.getString("INDEX_STATE"));
- assertFalse(rs.next());
-
- PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
- stmt.setString(1, "a");
- stmt.setString(2, "x");
- stmt.setString(3, "1");
- stmt.execute();
- conn.commit();
-
- // find a RS which doesn't has CATALOG table
- TableName catalogTable = TableName.valueOf("SYSTEM.CATALOG");
- TableName indexTable = TableName.valueOf(fullIndexName);
- final HBaseCluster cluster = this.getUtility().getHBaseCluster();
- Collection<ServerName> rss = cluster.getClusterStatus().getServers();
- HBaseAdmin admin = this.getUtility().getHBaseAdmin();
- List<HRegionInfo> regions = admin.getTableRegions(catalogTable);
- ServerName catalogRS = cluster.getServerHoldingRegion(regions.get(0).getTable(),
- regions.get(0).getRegionName());
- ServerName metaRS = cluster.getServerHoldingMeta();
- ServerName rsToBeKilled = null;
-
- // find first RS isn't holding META or CATALOG table
- for(ServerName curRS : rss) {
- if(!curRS.equals(catalogRS) && !metaRS.equals(curRS)) {
- rsToBeKilled = curRS;
- break;
- }
+ if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
+ break;
}
- assertTrue(rsToBeKilled != null);
-
- regions = admin.getTableRegions(indexTable);
- final HRegionInfo indexRegion = regions.get(0);
- final ServerName dstRS = rsToBeKilled;
- admin.move(indexRegion.getEncodedNameAsBytes(), Bytes.toBytes(rsToBeKilled.getServerName()));
- this.getUtility().waitFor(30000, 200, new Waiter.Predicate<Exception>() {
- @Override
- public boolean evaluate() throws Exception {
- ServerName sn = cluster.getServerHoldingRegion(indexRegion.getTable(),
- indexRegion.getRegionName());
- return (sn != null && sn.equals(dstRS));
- }
- });
-
- // use timer sending updates in every 10ms
- this.scheduleTimer = new Timer(true);
- this.scheduleTimer.schedule(new SendingUpdatesScheduleTask(conn, fullTableName), 0, 10);
- // let timer sending some updates
- Thread.sleep(100);
-
- // kill RS hosting index table
- this.getUtility().getHBaseCluster().killRegionServer(rsToBeKilled);
-
- // wait for index table completes recovery
- this.getUtility().waitUntilAllRegionsAssigned(indexTable);
-
- // Verify the metadata for index is correct.
- do {
- Thread.sleep(15 * 1000); // sleep 15 secs
- rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(TestUtil.DEFAULT_SCHEMA_NAME), indexName,
- new String[] { PTableType.INDEX.toString() });
- assertTrue(rs.next());
- if(PIndexState.ACTIVE.toString().equals(rs.getString("INDEX_STATE"))){
- break;
- }
- } while(true);
- this.scheduleTimer.cancel();
- }
+ } while(true);
+ this.scheduleTimer.cancel();
+ }
}
-
+
static class SendingUpdatesScheduleTask extends TimerTask {
private static final Log LOG = LogFactory.getLog(SendingUpdatesScheduleTask.class);
-
+
// inProgress is to prevent timer from invoking a new task while previous one is still
// running
private final static AtomicInteger inProgress = new AtomicInteger(0);
@@ -410,7 +412,7 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT {
if(inProgress.get() > 0){
return;
}
-
+
try {
inProgress.incrementAndGet();
inserts++;
@@ -427,5 +429,5 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT {
}
}
}
-
+
}
[35/50] [abbrv] phoenix git commit: PHOENIX-2552 Fix hanging Jenkins
builds
Posted by sa...@apache.org.
PHOENIX-2552 Fix hanging Jenkins builds
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/58465b31
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/58465b31
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/58465b31
Branch: refs/heads/4.x-HBase-1.0
Commit: 58465b31e929bae50f2ac6a33cabd2df1d82d554
Parents: 4cc7f3c
Author: James Taylor <jt...@salesforce.com>
Authored: Fri Jan 1 17:23:12 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Jan 1 17:24:32 2016 -0800
----------------------------------------------------------------------
.../main/java/org/apache/phoenix/memory/GlobalMemoryManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/58465b31/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java b/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
index e70b35f..18bc502 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
@@ -151,7 +151,7 @@ public class GlobalMemoryManager implements MemoryManager {
if (size > 0) {
logger.warn("Orphaned chunk of " + size + " bytes found during finalize");
}
- freeMemory();
+ //freeMemory();
} finally {
super.finalize();
}
[24/50] [abbrv] phoenix git commit: PHOENIX-2475 Date arithmetic
broken for time and timestamp (Ram)
Posted by sa...@apache.org.
PHOENIX-2475 Date arithmetic broken for time and timestamp (Ram)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/1b3a4ca4
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/1b3a4ca4
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/1b3a4ca4
Branch: refs/heads/4.x-HBase-1.0
Commit: 1b3a4ca41b104e64d7b9b4dd8cc10133d9458fba
Parents: 276069a
Author: ramkrishna <ra...@gmail.com>
Authored: Mon Dec 28 13:12:21 2015 +0530
Committer: ramkrishna <ra...@gmail.com>
Committed: Mon Dec 28 13:15:07 2015 +0530
----------------------------------------------------------------------
.../end2end/ClientTimeArithmeticQueryIT.java | 169 +++++++++++++++++++
.../phoenix/expression/DateAddExpression.java | 2 +-
.../expression/DateSubtractExpression.java | 2 +-
.../expression/TimestampAddExpression.java | 2 +-
.../expression/TimestampSubtractExpression.java | 2 +-
5 files changed, 173 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1b3a4ca4/phoenix-core/src/it/java/org/apache/phoenix/end2end/ClientTimeArithmeticQueryIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ClientTimeArithmeticQueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ClientTimeArithmeticQueryIT.java
index 00d835c..e617673 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ClientTimeArithmeticQueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ClientTimeArithmeticQueryIT.java
@@ -42,6 +42,8 @@ import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
+import java.sql.Time;
+import java.sql.Timestamp;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
@@ -638,5 +640,172 @@ public class ClientTimeArithmeticQueryIT extends BaseQueryIT {
conn.close();
}
}
+
+ @Test
+ public void testAddTimeStamp() throws Exception {
+ Connection conn;
+ PreparedStatement stmt;
+ ResultSet rs;
+
+ long ts = nextTimestamp();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement()
+ .execute(
+ "create table timestamp_table (ts timestamp primary key)");
+ conn.close();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30));
+ conn = DriverManager.getConnection(getUrl(), props);
+ stmt = conn.prepareStatement("upsert into timestamp_table values (?)");
+ stmt.setTimestamp(1, new Timestamp(1995 - 1900, 4, 2, 1, 1, 1, 1));
+ stmt.execute();
+ conn.commit();
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 60));
+ conn = DriverManager.getConnection(getUrl(), props);
+ rs = conn.createStatement().executeQuery("SELECT ts FROM timestamp_table");
+ assertTrue(rs.next());
+ assertEquals("1995-05-02 01:01:01.000000001",rs.getTimestamp(1).toString());
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 70));
+ conn = DriverManager.getConnection(getUrl(), props);
+ rs = conn.createStatement().executeQuery("SELECT ts + 1 FROM timestamp_table");
+ assertTrue(rs.next());
+ assertEquals("1995-05-03 01:01:01.000000001",rs.getTimestamp(1).toString());
+ }
+
+ @Test
+ public void testSubtractTimeStamp() throws Exception {
+ Connection conn;
+ PreparedStatement stmt;
+ ResultSet rs;
+
+ long ts = nextTimestamp();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement()
+ .execute(
+ "create table timestamp_table (ts timestamp primary key)");
+ conn.close();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30));
+ conn = DriverManager.getConnection(getUrl(), props);
+ stmt = conn.prepareStatement("upsert into timestamp_table values (?)");
+ stmt.setTimestamp(1, new Timestamp(1995 - 1900, 4, 2, 1, 1, 1, 1));
+ stmt.execute();
+ conn.commit();
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 60));
+ conn = DriverManager.getConnection(getUrl(), props);
+ rs = conn.createStatement().executeQuery("SELECT ts FROM timestamp_table");
+ assertTrue(rs.next());
+ assertEquals("1995-05-02 01:01:01.000000001",rs.getTimestamp(1).toString());
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 70));
+ conn = DriverManager.getConnection(getUrl(), props);
+ rs = conn.createStatement().executeQuery("SELECT ts - 1 FROM timestamp_table");
+ assertTrue(rs.next());
+ assertEquals("1995-05-01 01:01:01.000000001",rs.getTimestamp(1).toString());
+ }
+
+ @Test
+ public void testAddTime() throws Exception {
+ Connection conn;
+ PreparedStatement stmt;
+ ResultSet rs;
+
+ long ts = nextTimestamp();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement()
+ .execute(
+ "create table time_table (ts time primary key)");
+ conn.close();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30));
+ conn = DriverManager.getConnection(getUrl(), props);
+ stmt = conn.prepareStatement("upsert into time_table values (?)");
+ Time time = new Time(1995 - 1900, 4, 2);
+ stmt.setTime(1, time);
+ stmt.execute();
+ conn.commit();
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 60));
+ conn = DriverManager.getConnection(getUrl(), props);
+ rs = conn.createStatement().executeQuery("SELECT ts FROM time_table");
+ assertTrue(rs.next());
+ assertEquals(time.getTime(),rs.getTimestamp(1).getTime());
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 70));
+ conn = DriverManager.getConnection(getUrl(), props);
+ rs = conn.createStatement().executeQuery("SELECT ts + 1 FROM time_table");
+ assertTrue(rs.next());
+ assertEquals(time.getTime() + TestUtil.MILLIS_IN_DAY,rs.getTimestamp(1).getTime());
+ }
+
+ @Test
+ public void testSubtractTime() throws Exception {
+ Connection conn;
+ PreparedStatement stmt;
+ ResultSet rs;
+
+ long ts = nextTimestamp();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement()
+ .execute(
+ "create table time_table (ts time primary key)");
+ conn.close();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30));
+ conn = DriverManager.getConnection(getUrl(), props);
+ stmt = conn.prepareStatement("upsert into time_table values (?)");
+ Time time = new Time(1995 - 1900, 4, 2);
+ stmt.setTime(1, time);
+ stmt.execute();
+ conn.commit();
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 60));
+ conn = DriverManager.getConnection(getUrl(), props);
+ rs = conn.createStatement().executeQuery("SELECT ts FROM time_table");
+ assertTrue(rs.next());
+ assertEquals(time.getTime(),rs.getTimestamp(1).getTime());
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 70));
+ conn = DriverManager.getConnection(getUrl(), props);
+ rs = conn.createStatement().executeQuery("SELECT ts - 1 FROM time_table");
+ assertTrue(rs.next());
+ assertEquals(time.getTime() - TestUtil.MILLIS_IN_DAY,rs.getTimestamp(1).getTime());
+ }
+
+ @Test
+ public void testSubtractDate() throws Exception {
+ Connection conn;
+ PreparedStatement stmt;
+ ResultSet rs;
+
+ long ts = nextTimestamp();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement()
+ .execute(
+ "create table date_table (ts date primary key)");
+ conn.close();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30));
+ conn = DriverManager.getConnection(getUrl(), props);
+ stmt = conn.prepareStatement("upsert into date_table values (?)");
+ stmt.setDate(1, new Date(1995 - 1900, 4, 2));
+ stmt.execute();
+ conn.commit();
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 60));
+ conn = DriverManager.getConnection(getUrl(), props);
+ rs = conn.createStatement().executeQuery("SELECT ts FROM date_table");
+ assertTrue(rs.next());
+ assertEquals("1995-05-02",rs.getDate(1).toString());
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 70));
+ conn = DriverManager.getConnection(getUrl(), props);
+ rs = conn.createStatement().executeQuery("SELECT ts - 1 FROM date_table");
+ assertTrue(rs.next());
+ assertEquals("1995-05-01",rs.getDate(1).toString());
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1b3a4ca4/phoenix-core/src/main/java/org/apache/phoenix/expression/DateAddExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/DateAddExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/DateAddExpression.java
index 1174086..e9ffe90 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/DateAddExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/DateAddExpression.java
@@ -56,7 +56,7 @@ public class DateAddExpression extends AddExpression {
PDataType type = children.get(i).getDataType();
SortOrder sortOrder = children.get(i).getSortOrder();
if (type == PDecimal.INSTANCE) {
- BigDecimal bd = (BigDecimal) PDecimal.INSTANCE.toObject(ptr, sortOrder);
+ BigDecimal bd = (BigDecimal) PDecimal.INSTANCE.toObject(ptr, type, sortOrder);
value = bd.multiply(BD_MILLIS_IN_DAY).longValue();
} else if (type.isCoercibleTo(PLong.INSTANCE)) {
value = type.getCodec().decodeLong(ptr, sortOrder) * QueryConstants.MILLIS_IN_DAY;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1b3a4ca4/phoenix-core/src/main/java/org/apache/phoenix/expression/DateSubtractExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/DateSubtractExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/DateSubtractExpression.java
index 1d50c6a..2c0bec9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/DateSubtractExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/DateSubtractExpression.java
@@ -52,7 +52,7 @@ public class DateSubtractExpression extends SubtractExpression {
PDataType type = children.get(i).getDataType();
SortOrder sortOrder = children.get(i).getSortOrder();
if (type == PDecimal.INSTANCE) {
- BigDecimal bd = (BigDecimal) PDecimal.INSTANCE.toObject(ptr, sortOrder);
+ BigDecimal bd = (BigDecimal) PDecimal.INSTANCE.toObject(ptr, type, sortOrder);
value = bd.multiply(BD_MILLIS_IN_DAY).longValue();
} else if (type.isCoercibleTo(PLong.INSTANCE)) {
value = type.getCodec().decodeLong(ptr, sortOrder) * QueryConstants.MILLIS_IN_DAY;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1b3a4ca4/phoenix-core/src/main/java/org/apache/phoenix/expression/TimestampAddExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/TimestampAddExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/TimestampAddExpression.java
index f4c629b..6d1f214 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/TimestampAddExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/TimestampAddExpression.java
@@ -67,7 +67,7 @@ public class TimestampAddExpression extends AddExpression {
if(type == PTimestamp.INSTANCE || type == PUnsignedTimestamp.INSTANCE) {
value = (BigDecimal)(PDecimal.INSTANCE.toObject(ptr, type, sortOrder));
} else if (type.isCoercibleTo(PDecimal.INSTANCE)) {
- value = (((BigDecimal) PDecimal.INSTANCE.toObject(ptr, sortOrder)).multiply(QueryConstants.BD_MILLIS_IN_DAY)).setScale(6, RoundingMode.HALF_UP);
+ value = (((BigDecimal) PDecimal.INSTANCE.toObject(ptr, type, sortOrder)).multiply(QueryConstants.BD_MILLIS_IN_DAY)).setScale(6, RoundingMode.HALF_UP);
} else if (type.isCoercibleTo(PDouble.INSTANCE)) {
value = ((BigDecimal.valueOf(type.getCodec().decodeDouble(ptr, sortOrder))).multiply(QueryConstants.BD_MILLIS_IN_DAY)).setScale(6, RoundingMode.HALF_UP);
} else {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1b3a4ca4/phoenix-core/src/main/java/org/apache/phoenix/expression/TimestampSubtractExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/TimestampSubtractExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/TimestampSubtractExpression.java
index 87d634c..115bc16 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/TimestampSubtractExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/TimestampSubtractExpression.java
@@ -64,7 +64,7 @@ public class TimestampSubtractExpression extends SubtractExpression {
if(type == PTimestamp.INSTANCE || type == PUnsignedTimestamp.INSTANCE) {
value = (BigDecimal)(PDecimal.INSTANCE.toObject(ptr, type, sortOrder));
} else if (type.isCoercibleTo(PDecimal.INSTANCE)) {
- value = (((BigDecimal) PDecimal.INSTANCE.toObject(ptr, sortOrder)).multiply(BD_MILLIS_IN_DAY)).setScale(6, RoundingMode.HALF_UP);
+ value = (((BigDecimal) PDecimal.INSTANCE.toObject(ptr, type, sortOrder)).multiply(BD_MILLIS_IN_DAY)).setScale(6, RoundingMode.HALF_UP);
} else if (type.isCoercibleTo(PDouble.INSTANCE)) {
value = ((BigDecimal.valueOf(type.getCodec().decodeDouble(ptr, sortOrder))).multiply(BD_MILLIS_IN_DAY)).setScale(6, RoundingMode.HALF_UP);
} else {
[18/50] [abbrv] phoenix git commit: PHOENIX-2503 Multiple Java
NoClass/Method Errors with Spark and Phoenix
Posted by sa...@apache.org.
PHOENIX-2503 Multiple Java NoClass/Method Errors with Spark and Phoenix
The calcite dependency in the regular client JAR is pulling in a version
of com.fasterxml.jackson which is incompatible with the Spark runtime.
This patch creates a new assembly artifact, client-spark, which attempts
to include only the client JARs necessary for the Spark integration to
work.
Also made sure the Phoenix driver is explicitly registered in the PhoenixRDD
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7c76bbab
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7c76bbab
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7c76bbab
Branch: refs/heads/4.x-HBase-1.0
Commit: 7c76bbabdf308e67d548d613987dcc024d65bf39
Parents: b35cb98
Author: Josh Mahonin <jm...@gmail.com>
Authored: Mon Dec 21 10:30:36 2015 -0500
Committer: Josh Mahonin <jm...@gmail.com>
Committed: Mon Dec 21 10:38:00 2015 -0500
----------------------------------------------------------------------
phoenix-assembly/pom.xml | 17 ++++
phoenix-assembly/src/build/client-spark.xml | 87 ++++++++++++++++++++
.../org/apache/phoenix/spark/PhoenixRDD.scala | 6 ++
3 files changed, 110 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7c76bbab/phoenix-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/pom.xml b/phoenix-assembly/pom.xml
index 34a53dc..1945cac 100644
--- a/phoenix-assembly/pom.xml
+++ b/phoenix-assembly/pom.xml
@@ -64,6 +64,23 @@
</descriptors>
</configuration>
</execution>
+ <!-- Due to SPARK-8332 and Calcite's fasterxml dependency, we need a custom spark client -->
+ <execution>
+ <id>client-spark</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>phoenix-${project.version}</finalName>
+ <attach>false</attach>
+ <appendAssemblyId>true</appendAssemblyId>
+ <descriptors>
+ <!-- build the phoenix spark client jar -->
+ <descriptor>src/build/client-spark.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
<execution>
<id>client-minimal</id>
<phase>package</phase>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7c76bbab/phoenix-assembly/src/build/client-spark.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/client-spark.xml b/phoenix-assembly/src/build/client-spark.xml
new file mode 100644
index 0000000..748f1d8
--- /dev/null
+++ b/phoenix-assembly/src/build/client-spark.xml
@@ -0,0 +1,87 @@
+<?xml version='1.0'?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+<!-- Due to SPARK-8332 and Calcite's fasterxml dependency, we need a custom spark client -->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>client-spark</id>
+ <!-- All the dependencies (unpacked) necessary to run phoenix from a single, stand-alone jar -->
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <containerDescriptorHandlers>
+ <containerDescriptorHandler>
+ <!--
+ aggregate SPI's so that things like HDFS FileSystem works in uberjar
+ http://docs.oracle.com/javase/tutorial/sound/SPI-intro.html
+ -->
+ <handlerName>metaInf-services</handlerName>
+ </containerDescriptorHandler>
+ </containerDescriptorHandlers>
+
+ <componentDescriptors>
+ <componentDescriptor>src/build/components-minimal.xml</componentDescriptor>
+ </componentDescriptors>
+
+ <dependencySets>
+ <dependencySet>
+ <!-- Unpack all the dependencies to class files, since java doesn't support
+ jar of jars for running -->
+ <unpack>true</unpack>
+ <!-- save these dependencies to the top-level -->
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>org.apache.hbase:hbase*</include>
+ <include>org.apache.htrace:htrace-core</include>
+ <include>io.netty:netty-all</include>
+ <include>commons-codec:commons-codec</include>
+ <include>co.cask.tephra:tephra*</include>
+ <include>org.apache.twill:twill*</include>
+ <include>org.apache.thrift:*</include>
+ <include>com.google.code.gson:gson*</include>
+ <!-- We use a newer version of guava than HBase - this might be an issue? -->
+ <include>com.google.guava:guava</include>
+ <!-- HBase also pulls in these dependencies on its own, should we include-them? -->
+ <include>com.google.protobuf:protobuf-java</include>
+ <include>org.slf4j:slf4j-api</include>
+ <include>org.apache.zookeeper:zookeeper</include>
+ <include>log4j:log4j</include>
+ <include>org.apache.hadoop:hadoop*</include>
+ <include>commons-configuration:commons-configuration</include>
+ <include>commons-io:commons-io</include>
+ <include>commons-logging:commons-logging</include>
+ <include>commons-lang:commons-lang</include>
+ <include>commons-cli:commons-cli</include>
+ <include>org.apache.commons:commons-csv</include>
+ <include>org.codehaus.jackson:jackson-mapper-asl</include>
+ <include>org.codehaus.jackson:jackson-core-asl</include>
+ <include>commons-collections:commons-collections</include>
+ <include>joda-time:joda-time</include>
+ <include>org.jruby.joni:joni</include>
+ <include>org.jruby.jcodings:jcodings</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7c76bbab/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala b/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala
index fa36a1f..d79189b 100644
--- a/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala
+++ b/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala
@@ -13,9 +13,12 @@
*/
package org.apache.phoenix.spark
+import java.sql.DriverManager
+
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.{HBaseConfiguration, HConstants}
import org.apache.hadoop.io.NullWritable
+import org.apache.phoenix.jdbc.PhoenixDriver
import org.apache.phoenix.mapreduce.PhoenixInputFormat
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil
import org.apache.phoenix.schema.types._
@@ -32,6 +35,9 @@ class PhoenixRDD(sc: SparkContext, table: String, columns: Seq[String],
@transient conf: Configuration)
extends RDD[PhoenixRecordWritable](sc, Nil) with Logging {
+ // Make sure to register the Phoenix driver
+ DriverManager.registerDriver(new PhoenixDriver)
+
@transient lazy val phoenixConf = {
getPhoenixConfiguration
}
[08/50] [abbrv] phoenix git commit: PHOENIX-2249 SYSTEM.STATS not
update after region merge occurs.(Ankit Singhal)
Posted by sa...@apache.org.
PHOENIX-2249 SYSTEM.STATS not update after region merge occurs.(Ankit Singhal)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/42506f40
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/42506f40
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/42506f40
Branch: refs/heads/4.x-HBase-1.0
Commit: 42506f4052fb1c9a4b3822a946fee7a90b7fd35d
Parents: d478eb1
Author: Rajeshbabu Chintaguntla <ra...@apache.org>
Authored: Tue Dec 15 17:10:58 2015 +0530
Committer: Rajeshbabu Chintaguntla <ra...@apache.org>
Committed: Tue Dec 15 17:10:58 2015 +0530
----------------------------------------------------------------------
.../UngroupedAggregateRegionObserver.java | 15 ++++++++---
.../schema/stats/StatisticsCollector.java | 12 +++++----
.../phoenix/schema/stats/StatisticsScanner.java | 26 +++++++++++++++++---
3 files changed, 41 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/42506f40/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 4325760..a19cac9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -41,7 +41,9 @@ import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Mutation;
@@ -57,6 +59,7 @@ import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.io.WritableUtils;
import org.apache.phoenix.coprocessor.generated.PTableProtos;
import org.apache.phoenix.exception.DataExceedsCapacityException;
@@ -105,11 +108,11 @@ import org.apache.phoenix.util.TimeKeeper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import co.cask.tephra.TxConstants;
-
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import co.cask.tephra.TxConstants;
+
/**
* Region observer that aggregates ungrouped rows(i.e. SQL query with aggregation function and no GROUP BY).
@@ -627,6 +630,12 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
boolean useCurrentTime =
c.getEnvironment().getConfiguration().getBoolean(QueryServices.STATS_USE_CURRENT_TIME_ATTRIB,
QueryServicesOptions.DEFAULT_STATS_USE_CURRENT_TIME);
+ Connection conn = c.getEnvironment().getRegionServerServices().getConnection();
+ Pair<HRegionInfo, HRegionInfo> mergeRegions = null;
+ if (store.hasReferences()) {
+ mergeRegions = MetaTableAccessor.getRegionsFromMergeQualifier(conn,
+ c.getEnvironment().getRegion().getRegionInfo().getRegionName());
+ }
// Provides a means of clients controlling their timestamps to not use current time
// when background tasks are updating stats. Instead we track the max timestamp of
// the cells and use that.
@@ -634,7 +643,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
StatisticsCollector stats = new StatisticsCollector(
c.getEnvironment(), table.getNameAsString(),
clientTimeStamp, store.getFamily().getName());
- internalScanner = stats.createCompactionScanner(c.getEnvironment().getRegion(), store, scanner);
+ internalScanner = stats.createCompactionScanner(c.getEnvironment().getRegion(), store, scanner, mergeRegions);
} catch (IOException e) {
// If we can't reach the stats table, don't interrupt the normal
// compaction operation, just log a warning.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/42506f40/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
index 272cac6..22fdf90 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
@@ -26,6 +26,7 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.client.Mutation;
@@ -215,13 +216,14 @@ public class StatisticsCollector {
}
}
- public InternalScanner createCompactionScanner(HRegion region, Store store, InternalScanner s) throws IOException {
+ public InternalScanner createCompactionScanner(HRegion region, Store store, InternalScanner s,
+ Pair<HRegionInfo, HRegionInfo> mergeRegions) throws IOException {
// See if this is for Major compaction
if (logger.isDebugEnabled()) {
logger.debug("Compaction scanner created for stats");
}
ImmutableBytesPtr cfKey = new ImmutableBytesPtr(store.getFamily().getName());
- return getInternalScanner(region, store, s, cfKey);
+ return getInternalScanner(region, store, s, cfKey, mergeRegions);
}
public void splitStats(HRegion parent, HRegion left, HRegion right) {
@@ -243,9 +245,9 @@ public class StatisticsCollector {
}
}
- protected InternalScanner getInternalScanner(HRegion region, Store store,
- InternalScanner internalScan, ImmutableBytesPtr family) {
- return new StatisticsScanner(this, statsTable, region, internalScan, family);
+ protected InternalScanner getInternalScanner(HRegion region, Store store, InternalScanner internalScan,
+ ImmutableBytesPtr family, Pair<HRegionInfo, HRegionInfo> mergeRegions) {
+ return new StatisticsScanner(this, statsTable, region, internalScan, family, mergeRegions);
}
public void clear() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/42506f40/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
index de59304..5c460e3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
@@ -24,10 +24,12 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
/**
@@ -40,14 +42,16 @@ public class StatisticsScanner implements InternalScanner {
private HRegion region;
private StatisticsCollector tracker;
private ImmutableBytesPtr family;
+ private Pair<HRegionInfo, HRegionInfo> mergeRegions;
public StatisticsScanner(StatisticsCollector tracker, StatisticsWriter stats, HRegion region,
- InternalScanner delegate, ImmutableBytesPtr family) {
+ InternalScanner delegate, ImmutableBytesPtr family, Pair<HRegionInfo, HRegionInfo> mergeRegions) {
this.tracker = tracker;
this.stats = stats;
this.delegate = delegate;
this.region = region;
this.family = family;
+ this.mergeRegions = mergeRegions;
}
@Override
@@ -83,6 +87,22 @@ public class StatisticsScanner implements InternalScanner {
// update the statistics table
// Just verify if this if fine
ArrayList<Mutation> mutations = new ArrayList<Mutation>();
+ if (mergeRegions != null) {
+ if (mergeRegions.getFirst() != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Deleting stale stats for the region "
+ + mergeRegions.getFirst().getRegionNameAsString() + " as part of major compaction");
+ }
+ stats.deleteStats(mergeRegions.getFirst().getRegionName(), tracker, family, mutations);
+ }
+ if (mergeRegions.getSecond() != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Deleting stale stats for the region "
+ + mergeRegions.getSecond().getRegionNameAsString() + " as part of major compaction");
+ }
+ stats.deleteStats(mergeRegions.getSecond().getRegionName(), tracker, family, mutations);
+ }
+ }
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting the stats for the region " + region.getRegionNameAsString()
+ " as part of major compaction");
@@ -115,9 +135,7 @@ public class StatisticsScanner implements InternalScanner {
if (toThrow == null) toThrow = e;
LOG.error("Error while closing the scanner", e);
} finally {
- if (toThrow != null) {
- throw toThrow;
- }
+ if (toThrow != null) { throw toThrow; }
}
}
}
[07/50] [abbrv] phoenix git commit: PHOENIX-2505 Unexpected error
caused by GROUP BY
Posted by sa...@apache.org.
PHOENIX-2505 Unexpected error caused by GROUP BY
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d478eb10
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d478eb10
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d478eb10
Branch: refs/heads/4.x-HBase-1.0
Commit: d478eb10b6cc6d8d0d24f85ba85ef35e742a5f3c
Parents: ee813ce
Author: James Taylor <jt...@salesforce.com>
Authored: Sun Dec 13 17:12:20 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Dec 14 11:52:43 2015 -0800
----------------------------------------------------------------------
.../apache/phoenix/end2end/GroupByCaseIT.java | 49 ++++++++++++++++++++
1 file changed, 49 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d478eb10/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
index 64444da..0f1568c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
@@ -121,6 +121,55 @@ public class GroupByCaseIT extends BaseHBaseManagedTimeIT {
}
@Test
+ public void testBooleanInGroupBy() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String ddl = " create table bool_gb(id varchar primary key,v1 boolean, v2 integer, v3 integer)";
+
+ createTestTable(getUrl(), ddl);
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO bool_gb(id,v2,v3) VALUES(?,?,?)");
+ stmt.setString(1, "a");
+ stmt.setInt(2, 1);
+ stmt.setInt(3, 1);
+ stmt.execute();
+ stmt.close();
+ stmt = conn.prepareStatement("UPSERT INTO bool_gb VALUES(?,?,?,?)");
+ stmt.setString(1, "b");
+ stmt.setBoolean(2, false);
+ stmt.setInt(3, 2);
+ stmt.setInt(4, 2);
+ stmt.execute();
+ stmt.setString(1, "c");
+ stmt.setBoolean(2, true);
+ stmt.setInt(3, 3);
+ stmt.setInt(4, 3);
+ stmt.execute();
+ conn.commit();
+
+ String[] gbs = {"v1,v2,v3","v1,v3,v2","v2,v1,v3"};
+ for (String gb : gbs) {
+ ResultSet rs = conn.createStatement().executeQuery("SELECT v1, v2, v3 from bool_gb group by " + gb);
+ assertTrue(rs.next());
+ assertEquals(false,rs.getBoolean("v1"));
+ assertTrue(rs.wasNull());
+ assertEquals(1,rs.getInt("v2"));
+ assertEquals(1,rs.getInt("v3"));
+ assertTrue(rs.next());
+ assertEquals(false,rs.getBoolean("v1"));
+ assertFalse(rs.wasNull());
+ assertEquals(2,rs.getInt("v2"));
+ assertEquals(2,rs.getInt("v3"));
+ assertTrue(rs.next());
+ assertEquals(true,rs.getBoolean("v1"));
+ assertEquals(3,rs.getInt("v2"));
+ assertEquals(3,rs.getInt("v3"));
+ assertFalse(rs.next());
+ rs.close();
+ }
+ conn.close();
+ }
+
+ @Test
public void testScanUri() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props);
[25/50] [abbrv] phoenix git commit: PHOENIX-2538 - CsvBulkLoadTool
should return non-zero exit status if import fails
Posted by sa...@apache.org.
PHOENIX-2538 - CsvBulkLoadTool should return non-zero exit status if import fails
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/55b7adad
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/55b7adad
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/55b7adad
Branch: refs/heads/4.x-HBase-1.0
Commit: 55b7adadda89cbd84fa7fb45067ecf3dbc78f29c
Parents: 1b3a4ca
Author: ravimagham <ra...@bazaarvoice.com>
Authored: Tue Dec 29 10:17:05 2015 -0800
Committer: ravimagham <ra...@bazaarvoice.com>
Committed: Tue Dec 29 10:17:05 2015 -0800
----------------------------------------------------------------------
.../phoenix/mapreduce/CsvBulkLoadToolIT.java | 65 +++++++++++++---
.../phoenix/mapreduce/AbstractBulkLoadTool.java | 80 ++++++++++----------
2 files changed, 94 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/55b7adad/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
index 7daacb4..2970d56 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolIT.java
@@ -17,6 +17,14 @@
*/
package org.apache.phoenix.mapreduce;
+import static org.apache.phoenix.query.BaseTest.setUpConfigForMiniCluster;
+import static org.apache.phoenix.query.QueryServices.DATE_FORMAT_ATTRIB;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
import java.io.PrintWriter;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -28,6 +36,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.mapred.FileAlreadyExistsException;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.jdbc.PhoenixDriver;
import org.apache.phoenix.query.QueryServices;
@@ -36,18 +45,9 @@ import org.apache.phoenix.util.DateUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import static org.apache.phoenix.query.BaseTest.setUpConfigForMiniCluster;
-import static org.apache.phoenix.query.QueryServices.DATE_FORMAT_ATTRIB;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
@Category(NeedsOwnMiniClusterTest.class)
public class CsvBulkLoadToolIT {
@@ -321,4 +321,51 @@ public class CsvBulkLoadToolIT {
stmt.close();
}
+ @Test
+ public void testInvalidArguments() {
+ String tableName = "TABLE8";
+ CsvBulkLoadTool csvBulkLoadTool = new CsvBulkLoadTool();
+ csvBulkLoadTool.setConf(hbaseTestUtil.getConfiguration());
+ try {
+ csvBulkLoadTool.run(new String[] {
+ "--input", "/tmp/input4.csv",
+ "--table", tableName,
+ "--zookeeper", zkQuorum });
+ fail(String.format("Table %s not created, hence should fail",tableName));
+ } catch (Exception ex) {
+ assertTrue(ex instanceof IllegalArgumentException);
+ assertTrue(ex.getMessage().contains(String.format("Table %s not found", tableName)));
+ }
+ }
+
+ @Test
+ public void testAlreadyExistsOutputPath() {
+ String tableName = "TABLE9";
+ String outputPath = "/tmp/output/tabl9";
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + tableName + "(ID INTEGER NOT NULL PRIMARY KEY, "
+ + "FIRST_NAME VARCHAR, LAST_NAME VARCHAR)");
+
+ FileSystem fs = FileSystem.get(hbaseTestUtil.getConfiguration());
+ fs.create(new Path(outputPath));
+ FSDataOutputStream outputStream = fs.create(new Path("/tmp/input9.csv"));
+ PrintWriter printWriter = new PrintWriter(outputStream);
+ printWriter.println("1,FirstName 1,LastName 1");
+ printWriter.println("2,FirstName 2,LastName 2");
+ printWriter.close();
+
+ CsvBulkLoadTool csvBulkLoadTool = new CsvBulkLoadTool();
+ csvBulkLoadTool.setConf(hbaseTestUtil.getConfiguration());
+ csvBulkLoadTool.run(new String[] {
+ "--input", "/tmp/input9.csv",
+ "--output", outputPath,
+ "--table", tableName,
+ "--zookeeper", zkQuorum });
+
+ fail(String.format("Output path %s already exists. hence, should fail",outputPath));
+ } catch (Exception ex) {
+ assertTrue(ex instanceof FileAlreadyExistsException);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/55b7adad/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
index 1d2594d..4b5d618 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
@@ -17,10 +17,14 @@
*/
package org.apache.phoenix.mapreduce;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Splitter;
-import com.google.common.collect.Lists;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
@@ -57,14 +61,10 @@ import org.apache.phoenix.util.StringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ExecutionException;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
/**
* Base tool for running MapReduce-based ingests of data.
@@ -174,8 +174,7 @@ public abstract class AbstractBulkLoadTool extends Configured implements Tool {
return loadData(conf, cmdLine);
}
- private int loadData(Configuration conf, CommandLine cmdLine) throws SQLException,
- InterruptedException, ExecutionException, ClassNotFoundException {
+ private int loadData(Configuration conf, CommandLine cmdLine) throws Exception {
String tableName = cmdLine.getOptionValue(TABLE_NAME_OPT.getOpt());
String schemaName = cmdLine.getOptionValue(SCHEMA_NAME_OPT.getOpt());
String indexTableName = cmdLine.getOptionValue(INDEX_TABLE_NAME_OPT.getOpt());
@@ -255,47 +254,44 @@ public abstract class AbstractBulkLoadTool extends Configured implements Tool {
/**
* Submits the jobs to the cluster.
* Loads the HFiles onto the respective tables.
+ * @throws Exception
*/
public int submitJob(final Configuration conf, final String qualifiedTableName,
- final String inputPaths, final Path outputPath, List<TargetTableRef> tablesToBeLoaded) {
- try {
- Job job = new Job(conf, "Phoenix MapReduce import for " + qualifiedTableName);
- FileInputFormat.addInputPaths(job, inputPaths);
- FileOutputFormat.setOutputPath(job, outputPath);
+ final String inputPaths, final Path outputPath, List<TargetTableRef> tablesToBeLoaded) throws Exception {
+
+ Job job = Job.getInstance(conf, "Phoenix MapReduce import for " + qualifiedTableName);
+ FileInputFormat.addInputPaths(job, inputPaths);
+ FileOutputFormat.setOutputPath(job, outputPath);
- job.setInputFormatClass(TextInputFormat.class);
- job.setMapOutputKeyClass(TableRowkeyPair.class);
- job.setMapOutputValueClass(KeyValue.class);
- job.setOutputKeyClass(TableRowkeyPair.class);
- job.setOutputValueClass(KeyValue.class);
- job.setReducerClass(FormatToKeyValueReducer.class);
+ job.setInputFormatClass(TextInputFormat.class);
+ job.setMapOutputKeyClass(TableRowkeyPair.class);
+ job.setMapOutputValueClass(KeyValue.class);
+ job.setOutputKeyClass(TableRowkeyPair.class);
+ job.setOutputValueClass(KeyValue.class);
+ job.setReducerClass(FormatToKeyValueReducer.class);
- MultiHfileOutputFormat.configureIncrementalLoad(job, tablesToBeLoaded);
+ MultiHfileOutputFormat.configureIncrementalLoad(job, tablesToBeLoaded);
- final String tableNamesAsJson = TargetTableRefFunctions.NAMES_TO_JSON.apply(tablesToBeLoaded);
- job.getConfiguration().set(FormatToKeyValueMapper.TABLE_NAMES_CONFKEY,tableNamesAsJson);
+ final String tableNamesAsJson = TargetTableRefFunctions.NAMES_TO_JSON.apply(tablesToBeLoaded);
+ job.getConfiguration().set(FormatToKeyValueMapper.TABLE_NAMES_CONFKEY,tableNamesAsJson);
- // give subclasses their hook
- setupJob(job);
+ // give subclasses their hook
+ setupJob(job);
- LOG.info("Running MapReduce import job from {} to {}", inputPaths, outputPath);
- boolean success = job.waitForCompletion(true);
-
- if (success) {
- LOG.info("Loading HFiles from {}", outputPath);
- completebulkload(conf,outputPath,tablesToBeLoaded);
- }
+ LOG.info("Running MapReduce import job from {} to {}", inputPaths, outputPath);
+ boolean success = job.waitForCompletion(true);
+ if (success) {
+ LOG.info("Loading HFiles from {}", outputPath);
+ completebulkload(conf,outputPath,tablesToBeLoaded);
LOG.info("Removing output directory {}", outputPath);
- if (!FileSystem.get(conf).delete(outputPath, true)) {
- LOG.error("Removing output directory {} failed", outputPath);
+ if(!FileSystem.get(conf).delete(outputPath, true)) {
+ LOG.error("Failed to delete the output directory {}", outputPath);
}
return 0;
- } catch(Exception e) {
- LOG.error("Error occurred submitting BulkLoad ", e);
+ } else {
return -1;
}
-
}
private void completebulkload(Configuration conf,Path outputPath , List<TargetTableRef> tablesToBeLoaded) throws Exception {
[03/50] [abbrv] phoenix git commit: PHOENIX-2434 Improve booleans in
CSV import
Posted by sa...@apache.org.
PHOENIX-2434 Improve booleans in CSV import
Do not automatically treat any unrecognized input as being false,
but instead fail on it.
Also, recognize 1/0 and t/f as true/false.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b06a20c5
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b06a20c5
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b06a20c5
Branch: refs/heads/4.x-HBase-1.0
Commit: b06a20c55a61a14ea886f20106bb6a55640742e2
Parents: be0ccb4
Author: Bruno Dumon <br...@ngdata.com>
Authored: Sun Dec 13 16:04:07 2015 +0100
Committer: Gabriel Reid <ga...@ngdata.com>
Committed: Mon Dec 14 08:42:56 2015 +0100
----------------------------------------------------------------------
.../phoenix/util/csv/CsvUpsertExecutor.java | 15 +++++++++++++++
.../phoenix/util/AbstractUpsertExecutorTest.java | 18 +++++++++++++-----
.../phoenix/util/csv/CsvUpsertExecutorTest.java | 17 +++++++++++++++--
3 files changed, 43 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b06a20c5/phoenix-core/src/main/java/org/apache/phoenix/util/csv/CsvUpsertExecutor.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/csv/CsvUpsertExecutor.java b/phoenix-core/src/main/java/org/apache/phoenix/util/csv/CsvUpsertExecutor.java
index 4a3af21..cddafc6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/csv/CsvUpsertExecutor.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/csv/CsvUpsertExecutor.java
@@ -28,6 +28,7 @@ import javax.annotation.Nullable;
import org.apache.commons.csv.CSVRecord;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.types.PBoolean;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PTimestamp;
import org.apache.phoenix.util.ColumnInfo;
@@ -154,6 +155,20 @@ public class CsvUpsertExecutor extends UpsertExecutor<CSVRecord, String> {
byte[] byteValue = new byte[dataType.getByteSize()];
dataType.getCodec().encodeLong(epochTime, byteValue, 0);
return dataType.toObject(byteValue);
+ } else if (dataType == PBoolean.INSTANCE) {
+ switch (input.toLowerCase()) {
+ case "true":
+ case "t":
+ case "1":
+ return Boolean.TRUE;
+ case "false":
+ case "f":
+ case "0":
+ return Boolean.FALSE;
+ default:
+ throw new RuntimeException("Invalid boolean value: '" + input
+ + "', must be one of ['true','t','1','false','f','0']");
+ }
}
return dataType.toObject(input);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b06a20c5/phoenix-core/src/test/java/org/apache/phoenix/util/AbstractUpsertExecutorTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/AbstractUpsertExecutorTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/AbstractUpsertExecutorTest.java
index b614312..61b03fb 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/AbstractUpsertExecutorTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/AbstractUpsertExecutorTest.java
@@ -58,7 +58,8 @@ public abstract class AbstractUpsertExecutorTest<R, F> extends BaseConnectionles
new ColumnInfo("ID", Types.BIGINT),
new ColumnInfo("NAME", Types.VARCHAR),
new ColumnInfo("AGE", Types.INTEGER),
- new ColumnInfo("VALUES", PIntegerArray.INSTANCE.getSqlType()));
+ new ColumnInfo("VALUES", PIntegerArray.INSTANCE.getSqlType()),
+ new ColumnInfo("BEARD", Types.BOOLEAN));
preparedStatement = mock(PreparedStatement.class);
upsertListener = mock(UpsertExecutor.UpsertListener.class);
@@ -72,7 +73,8 @@ public abstract class AbstractUpsertExecutorTest<R, F> extends BaseConnectionles
@Test
public void testExecute() throws Exception {
- getUpsertExecutor().execute(createRecord(123L, "NameValue", 42, Arrays.asList(1, 2, 3)));
+ getUpsertExecutor().execute(createRecord(123L, "NameValue", 42,
+ Arrays.asList(1, 2, 3), true));
verify(upsertListener).upsertDone(1L);
verifyNoMoreInteractions(upsertListener);
@@ -81,6 +83,7 @@ public abstract class AbstractUpsertExecutorTest<R, F> extends BaseConnectionles
verify(preparedStatement).setObject(2, "NameValue");
verify(preparedStatement).setObject(3, Integer.valueOf(42));
verify(preparedStatement).setObject(4, PArrayDataType.instantiatePhoenixArray(PInteger.INSTANCE, new Object[]{1,2,3}));
+ verify(preparedStatement).setObject(5, Boolean.TRUE);
verify(preparedStatement).execute();
verifyNoMoreInteractions(preparedStatement);
}
@@ -96,7 +99,8 @@ public abstract class AbstractUpsertExecutorTest<R, F> extends BaseConnectionles
@Test
public void testExecute_TooManyFields() throws Exception {
- R recordWithTooManyFields = createRecord(123L, "NameValue", 42, Arrays.asList(1, 2, 3), "Garbage");
+ R recordWithTooManyFields = createRecord(123L, "NameValue", 42, Arrays.asList(1, 2, 3),
+ true, "Garbage");
getUpsertExecutor().execute(recordWithTooManyFields);
verify(upsertListener).upsertDone(1L);
@@ -106,13 +110,15 @@ public abstract class AbstractUpsertExecutorTest<R, F> extends BaseConnectionles
verify(preparedStatement).setObject(2, "NameValue");
verify(preparedStatement).setObject(3, Integer.valueOf(42));
verify(preparedStatement).setObject(4, PArrayDataType.instantiatePhoenixArray(PInteger.INSTANCE, new Object[]{1,2,3}));
+ verify(preparedStatement).setObject(5, Boolean.TRUE);
verify(preparedStatement).execute();
verifyNoMoreInteractions(preparedStatement);
}
@Test
public void testExecute_NullField() throws Exception {
- getUpsertExecutor().execute(createRecord(123L, "NameValue", null, Arrays.asList(1, 2, 3)));
+ getUpsertExecutor().execute(createRecord(123L, "NameValue", null,
+ Arrays.asList(1, 2, 3), false));
verify(upsertListener).upsertDone(1L);
verifyNoMoreInteractions(upsertListener);
@@ -121,13 +127,15 @@ public abstract class AbstractUpsertExecutorTest<R, F> extends BaseConnectionles
verify(preparedStatement).setObject(2, "NameValue");
verify(preparedStatement).setNull(3, columnInfoList.get(2).getSqlType());
verify(preparedStatement).setObject(4, PArrayDataType.instantiatePhoenixArray(PInteger.INSTANCE, new Object[]{1,2,3}));
+ verify(preparedStatement).setObject(5, Boolean.FALSE);
verify(preparedStatement).execute();
verifyNoMoreInteractions(preparedStatement);
}
@Test
public void testExecute_InvalidType() throws Exception {
- R recordWithInvalidType = createRecord(123L, "NameValue", "ThisIsNotANumber", Arrays.asList(1, 2, 3));
+ R recordWithInvalidType = createRecord(123L, "NameValue", "ThisIsNotANumber",
+ Arrays.asList(1, 2, 3), true);
getUpsertExecutor().execute(recordWithInvalidType);
verify(upsertListener).errorOnRecord(eq(recordWithInvalidType), any(Throwable.class));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b06a20c5/phoenix-core/src/test/java/org/apache/phoenix/util/csv/CsvUpsertExecutorTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/csv/CsvUpsertExecutorTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/csv/CsvUpsertExecutorTest.java
index 7a09bee..c887ff7 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/csv/CsvUpsertExecutorTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/csv/CsvUpsertExecutorTest.java
@@ -21,15 +21,20 @@ import java.io.IOException;
import java.sql.SQLException;
import java.util.List;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.phoenix.util.AbstractUpsertExecutorTest;
import org.apache.phoenix.util.UpsertExecutor;
import org.junit.Before;
+import org.junit.Test;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Iterables;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.verify;
public class CsvUpsertExecutorTest extends AbstractUpsertExecutorTest<CSVRecord, String> {
@@ -63,4 +68,12 @@ public class CsvUpsertExecutorTest extends AbstractUpsertExecutorTest<CSVRecord,
upsertExecutor = new CsvUpsertExecutor(conn, columnInfoList, preparedStatement,
upsertListener, ARRAY_SEP);
}
+
+ @Test
+ public void testExecute_InvalidBoolean() throws Exception {
+ CSVRecord csvRecordWithInvalidType = createRecord("123,NameValue,42,1:2:3,NotABoolean");
+ upsertExecutor.execute(ImmutableList.of(csvRecordWithInvalidType));
+
+ verify(upsertListener).errorOnRecord(eq(csvRecordWithInvalidType), any(Throwable.class));
+ }
}
[31/50] [abbrv] phoenix git commit: PHOENIX-2549 Modify Phoenix
DatabaseMetaData implementation for transaction support
Posted by sa...@apache.org.
PHOENIX-2549 Modify Phoenix DatabaseMetaData implementation for transaction support
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/fd5fff29
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/fd5fff29
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/fd5fff29
Branch: refs/heads/4.x-HBase-1.0
Commit: fd5fff29fee18b883827a031c5258589ee58019b
Parents: 9e66b35
Author: James Taylor <jt...@salesforce.com>
Authored: Wed Dec 30 10:43:30 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Dec 30 18:14:26 2015 -0800
----------------------------------------------------------------------
.../phoenix/exception/SQLExceptionCode.java | 1 +
.../apache/phoenix/jdbc/PhoenixConnection.java | 12 ++++--
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 44 +++++---------------
.../apache/phoenix/jdbc/PhoenixDriverTest.java | 27 ++++++++++++
4 files changed, 48 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/fd5fff29/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index 36036ae..228f06f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -282,6 +282,7 @@ public enum SQLExceptionCode {
CANNOT_ALTER_TO_BE_TXN_WITH_ROW_TIMESTAMP(1081, "44A12", "Cannot alter table to be transactional table if transactions are disabled"),
TX_MUST_BE_ENABLED_TO_SET_TX_CONTEXT(1082, "44A13", "Cannot set transaction context if transactions are disabled"),
TX_MUST_BE_ENABLED_TO_SET_AUTO_FLUSH(1083, "44A14", "Cannot set auto flush if transactions are disabled"),
+ TX_MUST_BE_ENABLED_TO_SET_ISOLATION_LEVEL(1084, "44A15", "Cannot set isolation level to TRANSACTION_READ_COMMITTED or TRANSACTION_SERIALIZABLE if transactions are disabled"),
/** Sequence related */
SEQUENCE_ALREADY_EXIST(1200, "42Z00", "Sequence already exists.", new Factory() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/fd5fff29/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
index 2d8810f..2a7b933 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
@@ -642,7 +642,10 @@ public class PhoenixConnection implements Connection, MetaDataMutated {
@Override
public int getTransactionIsolation() throws SQLException {
- return Connection.TRANSACTION_READ_COMMITTED;
+ boolean transactionsEnabled = getQueryServices().getProps().getBoolean(QueryServices.TRANSACTIONS_ENABLED,
+ QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED);
+ return transactionsEnabled ?
+ Connection.TRANSACTION_SERIALIZABLE : Connection.TRANSACTION_READ_COMMITTED;
}
@Override
@@ -806,8 +809,11 @@ public class PhoenixConnection implements Connection, MetaDataMutated {
@Override
public void setTransactionIsolation(int level) throws SQLException {
- if (level != Connection.TRANSACTION_READ_COMMITTED) {
- throw new SQLFeatureNotSupportedException();
+ boolean transactionsEnabled = getQueryServices().getProps().getBoolean(QueryServices.TRANSACTIONS_ENABLED,
+ QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED);
+ if (!transactionsEnabled && (level == Connection.TRANSACTION_REPEATABLE_READ || level == Connection.TRANSACTION_SERIALIZABLE)) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.TX_MUST_BE_ENABLED_TO_SET_ISOLATION_LEVEL)
+ .build().buildException();
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/fd5fff29/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index 87c02d0..a4748b3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -73,30 +73,8 @@ import com.google.common.collect.Lists;
/**
*
- * JDBC DatabaseMetaData implementation of Phoenix reflecting read-only nature of driver.
- * Supported metadata methods include:
- * {@link #getTables(String, String, String, String[])}
- * {@link #getColumns(String, String, String, String)}
- * {@link #getTableTypes()}
- * {@link #getPrimaryKeys(String, String, String)}
- * {@link #getIndexInfo(String, String, String, boolean, boolean)}
- * {@link #getSchemas()}
- * {@link #getSchemas(String, String)}
- * {@link #getDatabaseMajorVersion()}
- * {@link #getDatabaseMinorVersion()}
- * {@link #getClientInfoProperties()}
- * {@link #getConnection()}
- * {@link #getDatabaseProductName()}
- * {@link #getDatabaseProductVersion()}
- * {@link #getDefaultTransactionIsolation()}
- * {@link #getDriverName()}
- * {@link #getDriverVersion()}
- * {@link #getSuperTables(String, String, String)}
- * {@link #getCatalogs()}
- * Other ResultSet methods return an empty result set.
- *
- *
- * @since 0.1
+ * JDBC DatabaseMetaData implementation of Phoenix.
+ *
*/
public class PhoenixDatabaseMetaData implements DatabaseMetaData {
public static final int INDEX_NAME_INDEX = 4; // Shared with FAMILY_NAME_INDEX
@@ -1156,32 +1134,32 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
@Override
public boolean othersDeletesAreVisible(int type) throws SQLException {
- return true;
+ return false;
}
@Override
public boolean othersInsertsAreVisible(int type) throws SQLException {
- return true;
+ return false;
}
@Override
public boolean othersUpdatesAreVisible(int type) throws SQLException {
- return true;
+ return false;
}
@Override
public boolean ownDeletesAreVisible(int type) throws SQLException {
- return false;
+ return true;
}
@Override
public boolean ownInsertsAreVisible(int type) throws SQLException {
- return false;
+ return true;
}
@Override
public boolean ownUpdatesAreVisible(int type) throws SQLException {
- return false;
+ return true;
}
@Override
@@ -1387,7 +1365,7 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
@Override
public boolean supportsMultipleTransactions() throws SQLException {
- return false;
+ return true;
}
@Override
@@ -1534,12 +1512,12 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
@Override
public boolean supportsTransactionIsolationLevel(int level) throws SQLException {
- return level == connection.getTransactionIsolation();
+ return true;
}
@Override
public boolean supportsTransactions() throws SQLException {
- return false;
+ return true;
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/fd5fff29/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixDriverTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixDriverTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixDriverTest.java
index 61dc442..b70ea71 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixDriverTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixDriverTest.java
@@ -87,4 +87,31 @@ public class PhoenixDriverTest extends BaseConnectionlessQueryTest {
assertEquals(SQLExceptionCode.INVALID_SCN.getErrorCode(), e.getErrorCode());
}
}
+
+ @Test
+ public void testDisallowIsolationLevel() throws SQLException {
+ Connection conn = DriverManager.getConnection(getUrl());
+ conn.setTransactionIsolation(Connection.TRANSACTION_NONE);
+ conn.setTransactionIsolation(Connection.TRANSACTION_READ_UNCOMMITTED);
+ conn.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED);
+ try {
+ conn = DriverManager.getConnection(getUrl());
+ conn.setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
+ fail();
+ } catch(SQLException e) {
+ assertEquals(SQLExceptionCode.TX_MUST_BE_ENABLED_TO_SET_ISOLATION_LEVEL.getErrorCode(), e.getErrorCode());
+ }
+ try {
+ conn = DriverManager.getConnection(getUrl());
+ conn.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
+ fail();
+ } catch(SQLException e) {
+ assertEquals(SQLExceptionCode.TX_MUST_BE_ENABLED_TO_SET_ISOLATION_LEVEL.getErrorCode(), e.getErrorCode());
+ }
+ Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES);
+ props.setProperty(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
+ conn.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
+ }
}
[50/50] [abbrv] phoenix git commit: PHOENIX-2570 Start transaction
manager only when needed and reduce number of threads spawned for tests
Posted by sa...@apache.org.
PHOENIX-2570 Start transaction manager only when needed and reduce number of threads spawned for tests
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/5779f263
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/5779f263
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/5779f263
Branch: refs/heads/4.x-HBase-1.0
Commit: 5779f263c66033f4fb04615b23a6bf943a4afb7f
Parents: 5fb51ca
Author: Samarth <sa...@salesforce.com>
Authored: Wed Jan 6 15:16:46 2016 -0800
Committer: Samarth <sa...@salesforce.com>
Committed: Wed Jan 6 15:16:46 2016 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/query/QueryServices.java | 3 ++
.../phoenix/query/QueryServicesOptions.java | 26 ++++++++++++++++
.../java/org/apache/phoenix/query/BaseTest.java | 32 +++++++++-----------
.../phoenix/query/QueryServicesTestImpl.java | 12 +++++++-
pom.xml | 2 +-
5 files changed, 56 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5779f263/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 5fd7403..c0f4c4f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -201,6 +201,9 @@ public interface QueryServices extends SQLCloseable {
public static final String RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS = "phoenix.scanner.lease.renew.interval";
public static final String RENEW_LEASE_THRESHOLD_MILLISECONDS = "phoenix.scanner.lease.threshold";
public static final String RENEW_LEASE_THREAD_POOL_SIZE = "phoenix.scanner.lease.pool.size";
+ public static final String HCONNECTION_POOL_CORE_SIZE = "hbase.hconnection.threads.core";
+ public static final String HCONNECTION_POOL_MAX_SIZE = "hbase.hconnection.threads.max";
+ public static final String HTABLE_MAX_THREADS = "hbase.htable.threads.max";
/**
* Get executor service used for parallel scans
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5779f263/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index def0a18..efe6a70 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -616,4 +616,30 @@ public class QueryServicesOptions {
config.setBoolean(RENEW_LEASE_ENABLED, enable);
return this;
}
+
+ public QueryServicesOptions setIndexHandlerCount(int count) {
+ config.setInt(QueryServices.INDEX_HANDLER_COUNT_ATTRIB, count);
+ return this;
+ }
+
+ public QueryServicesOptions setMetadataHandlerCount(int count) {
+ config.setInt(QueryServices.METADATA_HANDLER_COUNT_ATTRIB, count);
+ return this;
+ }
+
+ public QueryServicesOptions setHConnectionPoolCoreSize(int count) {
+ config.setInt(QueryServices.HCONNECTION_POOL_CORE_SIZE, count);
+ return this;
+ }
+
+ public QueryServicesOptions setHConnectionPoolMaxSize(int count) {
+ config.setInt(QueryServices.HCONNECTION_POOL_MAX_SIZE, count);
+ return this;
+ }
+
+ public QueryServicesOptions setMaxThreadsPerHTable(int count) {
+ config.setInt(QueryServices.HTABLE_MAX_THREADS, count);
+ return this;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5779f263/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index 3f16065..be776b3 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -112,6 +112,12 @@ import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
+import co.cask.tephra.TransactionManager;
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.distributed.TransactionService;
+import co.cask.tephra.metrics.TxMetricsCollector;
+import co.cask.tephra.persist.InMemoryTransactionStateStorage;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -128,6 +134,7 @@ import org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory;
import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
import org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory;
import org.apache.hadoop.hbase.master.LoadBalancer;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.LocalIndexMerger;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.util.Bytes;
@@ -163,12 +170,6 @@ import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.distributed.TransactionService;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.persist.InMemoryTransactionStateStorage;
-
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -549,7 +550,6 @@ public abstract class BaseTest {
if (!clusterInitialized) {
url = setUpTestCluster(config, overrideProps);
clusterInitialized = true;
- setupTxManager();
}
return url;
}
@@ -573,6 +573,7 @@ public abstract class BaseTest {
assertTrue(destroyDriver(driver));
} finally {
driver = null;
+ teardownTxManager();
}
}
}
@@ -594,12 +595,8 @@ public abstract class BaseTest {
utility.shutdownMiniCluster();
}
} finally {
- try {
- teardownTxManager();
- } finally {
- utility = null;
- clusterInitialized = false;
- }
+ utility = null;
+ clusterInitialized = false;
}
}
}
@@ -612,6 +609,9 @@ public abstract class BaseTest {
String url = checkClusterInitialized(serverProps);
if (driver == null) {
driver = initAndRegisterDriver(url, clientProps);
+ if (clientProps.getBoolean(QueryServices.TRANSACTIONS_ENABLED, QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED)) {
+ setupTxManager();
+ }
}
}
@@ -712,9 +712,6 @@ public abstract class BaseTest {
conf.setInt("dfs.datanode.handler.count", 2);
conf.setInt("ipc.server.read.threadpool.size", 2);
conf.setInt("ipc.server.handler.threadpool.size", 2);
- conf.setInt("hbase.hconnection.threads.max", 2);
- conf.setInt("hbase.hconnection.threads.core", 2);
- conf.setInt("hbase.htable.threads.max", 2);
conf.setInt("hbase.regionserver.hlog.syncer.count", 2);
conf.setInt("hbase.hlog.asyncer.number", 2);
conf.setInt("hbase.assignment.zkevent.workers", 5);
@@ -734,7 +731,8 @@ public abstract class BaseTest {
if (oldDriver != newDriver) {
destroyDriver(oldDriver);
}
- Connection conn = newDriver.connect(url, PropertiesUtil.deepCopy(TEST_PROPERTIES));
+ Properties driverProps = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = newDriver.connect(url, driverProps);
conn.close();
return newDriver;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5779f263/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
index 037e8e7..8ff9a69 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
@@ -58,6 +58,11 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
private static final boolean DEFAULT_RUN_UPDATE_STATS_ASYNC = false;
private static final boolean DEFAULT_COMMIT_STATS_ASYNC = false;
public static final boolean DEFAULT_RENEW_LEASE_ENABLED = false;
+ public static final int DEFAULT_INDEX_HANDLER_COUNT = 5;
+ public static final int DEFAULT_METADATA_HANDLER_COUNT = 5;
+ public static final int DEFAULT_HCONNECTION_POOL_CORE_SIZE = 10;
+ public static final int DEFAULT_HCONNECTION_POOL_MAX_SIZE = 10;
+ public static final int DEFAULT_HTABLE_MAX_THREADS = 10;
/**
@@ -99,7 +104,12 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
.setExtraJDBCArguments(DEFAULT_EXTRA_JDBC_ARGUMENTS)
.setRunUpdateStatsAsync(DEFAULT_RUN_UPDATE_STATS_ASYNC)
.setCommitStatsAsync(DEFAULT_COMMIT_STATS_ASYNC)
- .setEnableRenewLease(DEFAULT_RENEW_LEASE_ENABLED);
+ .setEnableRenewLease(DEFAULT_RENEW_LEASE_ENABLED)
+ .setIndexHandlerCount(DEFAULT_INDEX_HANDLER_COUNT)
+ .setMetadataHandlerCount(DEFAULT_METADATA_HANDLER_COUNT)
+ .setHConnectionPoolCoreSize(DEFAULT_HCONNECTION_POOL_CORE_SIZE)
+ .setHConnectionPoolMaxSize(DEFAULT_HCONNECTION_POOL_MAX_SIZE)
+ .setMaxThreadsPerHTable(DEFAULT_HTABLE_MAX_THREADS);
}
public QueryServicesTestImpl(ReadOnlyProps defaultProps, ReadOnlyProps overrideProps) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5779f263/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index eebba33..2a9c438 100644
--- a/pom.xml
+++ b/pom.xml
@@ -130,7 +130,7 @@
<!-- Plugin options -->
<numForkedUT>3</numForkedUT>
- <numForkedIT>5</numForkedIT>
+ <numForkedIT>7</numForkedIT>
<!-- Set default encoding so multi-byte tests work correctly on the Mac -->
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
[43/50] [abbrv] phoenix git commit: PHOENIX-2556 Subqueries with
nested joins may not free hash cache
Posted by sa...@apache.org.
PHOENIX-2556 Subqueries with nested joins may not free hash cache
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a593e98f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a593e98f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a593e98f
Branch: refs/heads/4.x-HBase-1.0
Commit: a593e98fcd80804c00f2a95f51512f3661d1f18f
Parents: a7788da
Author: maryannxue <ma...@gmail.com>
Authored: Tue Jan 5 09:36:03 2016 -0500
Committer: maryannxue <ma...@gmail.com>
Committed: Tue Jan 5 09:36:03 2016 -0500
----------------------------------------------------------------------
.../ConnectionQueryServicesTestImpl.java | 5 +-
.../org/apache/phoenix/end2end/SubqueryIT.java | 48 ++++++++++++++++----
.../apache/phoenix/compile/DeleteCompiler.java | 36 ++++++++-------
.../apache/phoenix/execute/HashJoinPlan.java | 40 ++++++++--------
4 files changed, 85 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a593e98f/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
index f5d7f18..2d0ed60 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionQueryServicesTestImpl.java
@@ -17,6 +17,8 @@
*/
package org.apache.phoenix.end2end;
+import static org.junit.Assert.assertEquals;
+
import java.sql.SQLException;
import java.util.Properties;
import java.util.Set;
@@ -82,8 +84,7 @@ public class ConnectionQueryServicesTestImpl extends ConnectionQueryServicesImpl
this.connections = Sets.newHashSet();
SQLCloseables.closeAll(connections);
long unfreedBytes = clearCache();
- // FIXME: once PHOENIX-2556 is fixed, comment this back in
- // assertEquals(0,unfreedBytes);
+ assertEquals("Found unfreed bytes in server-side cache", 0, unfreedBytes);
} finally {
super.close();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a593e98f/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryIT.java
index 794c4f5..90ce327 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryIT.java
@@ -45,12 +45,14 @@ import java.util.Map;
import java.util.Properties;
import java.util.regex.Pattern;
+import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.TableAlreadyExistsException;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -366,6 +368,17 @@ public class SubqueryIT extends BaseHBaseManagedTimeIT {
}
}
+ @After
+ public void assertNoUnfreedMemory() throws SQLException {
+ Connection conn = DriverManager.getConnection(getUrl());
+ try {
+ long unfreedBytes = conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache();
+ assertEquals("Found bytes not freed on server side", 0, unfreedBytes);
+ } finally {
+ conn.close();
+ }
+ }
+
@Test
public void testNonCorrelatedSubquery() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
@@ -628,9 +641,12 @@ public class SubqueryIT extends BaseHBaseManagedTimeIT {
@Test
public void testComparisonSubquery() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
+ final Connection conn = DriverManager.getConnection(getUrl(), props);
try {
- String query = "SELECT \"order_id\", name FROM " + JOIN_ORDER_TABLE_FULL_NAME + " o JOIN " + JOIN_ITEM_TABLE_FULL_NAME + " i ON o.\"item_id\" = i.\"item_id\" WHERE quantity = (SELECT max(quantity) FROM " + JOIN_ORDER_TABLE_FULL_NAME + " q WHERE o.\"item_id\" = q.\"item_id\")";
+ String query = "SELECT \"order_id\", name FROM " + JOIN_ORDER_TABLE_FULL_NAME +
+ " o JOIN " + JOIN_ITEM_TABLE_FULL_NAME +
+ " i ON o.\"item_id\" = i.\"item_id\" WHERE quantity = (SELECT max(quantity) FROM " +
+ JOIN_ORDER_TABLE_FULL_NAME + " q WHERE o.\"item_id\" = q.\"item_id\")";
PreparedStatement statement = conn.prepareStatement(query);
ResultSet rs = statement.executeQuery();
assertTrue (rs.next());
@@ -648,7 +664,11 @@ public class SubqueryIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
- query = "SELECT \"order_id\", name FROM " + JOIN_ORDER_TABLE_FULL_NAME + " o JOIN " + JOIN_ITEM_TABLE_FULL_NAME + " i ON o.\"item_id\" = i.\"item_id\" WHERE quantity = (SELECT max(quantity) FROM " + JOIN_ITEM_TABLE_FULL_NAME + " i2 JOIN " + JOIN_ORDER_TABLE_FULL_NAME + " q ON i2.\"item_id\" = q.\"item_id\" WHERE o.\"item_id\" = i2.\"item_id\")";
+ query = "SELECT \"order_id\", name FROM " + JOIN_ORDER_TABLE_FULL_NAME +
+ " o JOIN " + JOIN_ITEM_TABLE_FULL_NAME +
+ " i ON o.\"item_id\" = i.\"item_id\" WHERE quantity = (SELECT max(quantity) FROM " +
+ JOIN_ITEM_TABLE_FULL_NAME + " i2 JOIN " + JOIN_ORDER_TABLE_FULL_NAME +
+ " q ON i2.\"item_id\" = q.\"item_id\" WHERE o.\"item_id\" = i2.\"item_id\")";
statement = conn.prepareStatement(query);
rs = statement.executeQuery();
assertTrue (rs.next());
@@ -666,7 +686,11 @@ public class SubqueryIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
- query = "SELECT name from " + JOIN_CUSTOMER_TABLE_FULL_NAME + " WHERE \"customer_id\" IN (SELECT \"customer_id\" FROM " + JOIN_ITEM_TABLE_FULL_NAME + " i JOIN " + JOIN_ORDER_TABLE_FULL_NAME + " o ON o.\"item_id\" = i.\"item_id\" WHERE i.name = 'T2' OR quantity > (SELECT avg(quantity) FROM " + JOIN_ORDER_TABLE_FULL_NAME + " q WHERE o.\"item_id\" = q.\"item_id\"))";
+ query = "SELECT name from " + JOIN_CUSTOMER_TABLE_FULL_NAME +
+ " WHERE \"customer_id\" IN (SELECT \"customer_id\" FROM " +
+ JOIN_ITEM_TABLE_FULL_NAME + " i JOIN " + JOIN_ORDER_TABLE_FULL_NAME +
+ " o ON o.\"item_id\" = i.\"item_id\" WHERE i.name = 'T2' OR quantity > (SELECT avg(quantity) FROM " +
+ JOIN_ORDER_TABLE_FULL_NAME + " q WHERE o.\"item_id\" = q.\"item_id\"))";
statement = conn.prepareStatement(query);
rs = statement.executeQuery();
assertTrue (rs.next());
@@ -680,7 +704,9 @@ public class SubqueryIT extends BaseHBaseManagedTimeIT {
String plan = QueryUtil.getExplainPlan(rs);
assertTrue("\"" + plan + "\" does not match \"" + plans[4] + "\"", Pattern.matches(plans[4], plan));
- query = "SELECT \"order_id\" FROM " + JOIN_ORDER_TABLE_FULL_NAME + " o WHERE quantity = (SELECT quantity FROM " + JOIN_ORDER_TABLE_FULL_NAME + " WHERE o.\"item_id\" = \"item_id\" AND \"order_id\" != '000000000000004')";
+ query = "SELECT \"order_id\" FROM " + JOIN_ORDER_TABLE_FULL_NAME +
+ " o WHERE quantity = (SELECT quantity FROM " + JOIN_ORDER_TABLE_FULL_NAME +
+ " WHERE o.\"item_id\" = \"item_id\" AND \"order_id\" != '000000000000004')";
statement = conn.prepareStatement(query);
rs = statement.executeQuery();
assertTrue (rs.next());
@@ -694,7 +720,9 @@ public class SubqueryIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
- query = "SELECT \"order_id\" FROM " + JOIN_ORDER_TABLE_FULL_NAME + " o WHERE quantity = (SELECT quantity FROM " + JOIN_ORDER_TABLE_FULL_NAME + " WHERE o.\"item_id\" = \"item_id\" AND \"order_id\" != '000000000000003')";
+ query = "SELECT \"order_id\" FROM " + JOIN_ORDER_TABLE_FULL_NAME +
+ " o WHERE quantity = (SELECT quantity FROM " + JOIN_ORDER_TABLE_FULL_NAME +
+ " WHERE o.\"item_id\" = \"item_id\" AND \"order_id\" != '000000000000003')";
statement = conn.prepareStatement(query);
rs = statement.executeQuery();
try {
@@ -703,7 +731,9 @@ public class SubqueryIT extends BaseHBaseManagedTimeIT {
} catch (SQLException e) {
}
- query = "SELECT \"order_id\" FROM " + JOIN_ORDER_TABLE_FULL_NAME + " o WHERE quantity = (SELECT max(quantity) FROM " + JOIN_ORDER_TABLE_FULL_NAME + " WHERE o.\"item_id\" = \"item_id\" AND \"order_id\" != '000000000000004' GROUP BY \"order_id\")";
+ query = "SELECT \"order_id\" FROM " + JOIN_ORDER_TABLE_FULL_NAME +
+ " o WHERE quantity = (SELECT max(quantity) FROM " + JOIN_ORDER_TABLE_FULL_NAME +
+ " WHERE o.\"item_id\" = \"item_id\" AND \"order_id\" != '000000000000004' GROUP BY \"order_id\")";
statement = conn.prepareStatement(query);
rs = statement.executeQuery();
assertTrue (rs.next());
@@ -717,7 +747,9 @@ public class SubqueryIT extends BaseHBaseManagedTimeIT {
assertFalse(rs.next());
- query = "SELECT \"order_id\" FROM " + JOIN_ORDER_TABLE_FULL_NAME + " o WHERE quantity = (SELECT max(quantity) FROM " + JOIN_ORDER_TABLE_FULL_NAME + " WHERE o.\"item_id\" = \"item_id\" AND \"order_id\" != '000000000000003' GROUP BY \"order_id\")";
+ query = "SELECT \"order_id\" FROM " + JOIN_ORDER_TABLE_FULL_NAME +
+ " o WHERE quantity = (SELECT max(quantity) FROM " + JOIN_ORDER_TABLE_FULL_NAME +
+ " WHERE o.\"item_id\" = \"item_id\" AND \"order_id\" != '000000000000003' GROUP BY \"order_id\")";
statement = conn.prepareStatement(query);
rs = statement.executeQuery();
try {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a593e98f/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
index ac9f7d1..924ed43 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
@@ -618,23 +618,27 @@ public class DeleteCompiler {
@Override
public MutationState execute() throws SQLException {
ResultIterator iterator = plan.iterator();
- if (!hasLimit) {
- Tuple tuple;
- long totalRowCount = 0;
- while ((tuple=iterator.next()) != null) {// Runs query
- Cell kv = tuple.getValue(0);
- totalRowCount += PLong.INSTANCE.getCodec().decodeLong(kv.getValueArray(), kv.getValueOffset(), SortOrder.getDefault());
+ try {
+ if (!hasLimit) {
+ Tuple tuple;
+ long totalRowCount = 0;
+ while ((tuple=iterator.next()) != null) {// Runs query
+ Cell kv = tuple.getValue(0);
+ totalRowCount += PLong.INSTANCE.getCodec().decodeLong(kv.getValueArray(), kv.getValueOffset(), SortOrder.getDefault());
+ }
+ // Return total number of rows that have been delete. In the case of auto commit being off
+ // the mutations will all be in the mutation state of the current connection.
+ MutationState state = new MutationState(maxSize, connection, totalRowCount);
+
+ // set the read metrics accumulated in the parent context so that it can be published when the mutations are committed.
+ state.setReadMetricQueue(plan.getContext().getReadMetricsQueue());
+
+ return state;
+ } else {
+ return deleteRows(plan.getContext(), tableRef, deleteFromImmutableIndexToo ? plan.getTableRef() : null, iterator, plan.getProjector(), plan.getTableRef());
}
- // Return total number of rows that have been delete. In the case of auto commit being off
- // the mutations will all be in the mutation state of the current connection.
- MutationState state = new MutationState(maxSize, connection, totalRowCount);
-
- // set the read metrics accumulated in the parent context so that it can be published when the mutations are committed.
- state.setReadMetricQueue(plan.getContext().getReadMetricsQueue());
-
- return state;
- } else {
- return deleteRows(plan.getContext(), tableRef, deleteFromImmutableIndexToo ? plan.getTableRef() : null, iterator, plan.getProjector(), plan.getTableRef());
+ } finally {
+ iterator.close();
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a593e98f/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
index cf89380..053dc2b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
@@ -88,7 +88,7 @@ public class HashJoinPlan extends DelegateQueryPlan {
private final boolean recompileWhereClause;
private final Set<TableRef> tableRefs;
private final int maxServerCacheTimeToLive;
- private List<SQLCloseable> dependencies;
+ private final List<SQLCloseable> dependencies = Lists.newArrayList();
private HashCacheClient hashClient;
private AtomicLong firstJobEndTime;
private List<Expression> keyRangeExpressions;
@@ -96,7 +96,7 @@ public class HashJoinPlan extends DelegateQueryPlan {
public static HashJoinPlan create(SelectStatement statement,
QueryPlan plan, HashJoinInfo joinInfo, SubPlan[] subPlans) {
if (!(plan instanceof HashJoinPlan))
- return new HashJoinPlan(statement, plan, joinInfo, subPlans, joinInfo == null);
+ return new HashJoinPlan(statement, plan, joinInfo, subPlans, joinInfo == null, Collections.<SQLCloseable>emptyList());
HashJoinPlan hashJoinPlan = (HashJoinPlan) plan;
assert (hashJoinPlan.joinInfo == null && hashJoinPlan.delegate instanceof BaseQueryPlan);
@@ -108,12 +108,13 @@ public class HashJoinPlan extends DelegateQueryPlan {
for (SubPlan subPlan : subPlans) {
mergedSubPlans[i++] = subPlan;
}
- return new HashJoinPlan(statement, hashJoinPlan.delegate, joinInfo, mergedSubPlans, true);
+ return new HashJoinPlan(statement, hashJoinPlan.delegate, joinInfo, mergedSubPlans, true, hashJoinPlan.dependencies);
}
private HashJoinPlan(SelectStatement statement,
- QueryPlan plan, HashJoinInfo joinInfo, SubPlan[] subPlans, boolean recompileWhereClause) {
+ QueryPlan plan, HashJoinInfo joinInfo, SubPlan[] subPlans, boolean recompileWhereClause, List<SQLCloseable> dependencies) {
super(plan);
+ this.dependencies.addAll(dependencies);
this.statement = statement;
this.joinInfo = joinInfo;
this.subPlans = subPlans;
@@ -143,8 +144,7 @@ public class HashJoinPlan extends DelegateQueryPlan {
PhoenixConnection connection = getContext().getConnection();
ConnectionQueryServices services = connection.getQueryServices();
ExecutorService executor = services.getExecutor();
- List<Future<Object>> futures = Lists.<Future<Object>>newArrayListWithExpectedSize(count);
- dependencies = Lists.newArrayList();
+ List<Future<ServerCache>> futures = Lists.newArrayListWithExpectedSize(count);
if (joinInfo != null) {
hashClient = hashClient != null ?
hashClient
@@ -155,11 +155,12 @@ public class HashJoinPlan extends DelegateQueryPlan {
for (int i = 0; i < count; i++) {
final int index = i;
- futures.add(executor.submit(new JobCallable<Object>() {
+ futures.add(executor.submit(new JobCallable<ServerCache>() {
@Override
- public Object call() throws Exception {
- return subPlans[index].execute(HashJoinPlan.this);
+ public ServerCache call() throws Exception {
+ ServerCache cache = subPlans[index].execute(HashJoinPlan.this);
+ return cache;
}
@Override
@@ -177,7 +178,10 @@ public class HashJoinPlan extends DelegateQueryPlan {
SQLException firstException = null;
for (int i = 0; i < count; i++) {
try {
- Object result = futures.get(i).get();
+ ServerCache result = futures.get(i).get();
+ if (result != null) {
+ dependencies.add(result);
+ }
subPlans[i].postProcess(result, this);
} catch (InterruptedException e) {
if (firstException == null) {
@@ -261,8 +265,8 @@ public class HashJoinPlan extends DelegateQueryPlan {
}
protected interface SubPlan {
- public Object execute(HashJoinPlan parent) throws SQLException;
- public void postProcess(Object result, HashJoinPlan parent) throws SQLException;
+ public ServerCache execute(HashJoinPlan parent) throws SQLException;
+ public void postProcess(ServerCache result, HashJoinPlan parent) throws SQLException;
public List<String> getPreSteps(HashJoinPlan parent) throws SQLException;
public List<String> getPostSteps(HashJoinPlan parent) throws SQLException;
public QueryPlan getInnerPlan();
@@ -280,7 +284,7 @@ public class HashJoinPlan extends DelegateQueryPlan {
}
@Override
- public Object execute(HashJoinPlan parent) throws SQLException {
+ public ServerCache execute(HashJoinPlan parent) throws SQLException {
List<Object> values = Lists.<Object> newArrayList();
ResultIterator iterator = plan.iterator();
RowProjector projector = plan.getProjector();
@@ -319,7 +323,7 @@ public class HashJoinPlan extends DelegateQueryPlan {
}
@Override
- public void postProcess(Object result, HashJoinPlan parent) throws SQLException {
+ public void postProcess(ServerCache result, HashJoinPlan parent) throws SQLException {
}
@Override
@@ -365,7 +369,7 @@ public class HashJoinPlan extends DelegateQueryPlan {
}
@Override
- public Object execute(HashJoinPlan parent) throws SQLException {
+ public ServerCache execute(HashJoinPlan parent) throws SQLException {
ScanRanges ranges = parent.delegate.getContext().getScanRanges();
List<Expression> keyRangeRhsValues = null;
if (keyRangeRhsExpression != null) {
@@ -387,6 +391,7 @@ public class HashJoinPlan extends DelegateQueryPlan {
// Evaluate key expressions for hash join key range optimization.
keyRangeRhsValues.add(HashCacheClient.evaluateKeyExpression(keyRangeRhsExpression, result, plan.getContext().getTempPtr()));
}
+ iterator.close();
}
if (keyRangeRhsValues != null) {
parent.keyRangeExpressions.add(parent.createKeyRangeExpression(keyRangeLhsExpression, keyRangeRhsExpression, keyRangeRhsValues, plan.getContext().getTempPtr(), plan.getContext().getCurrentTable().getTable().rowKeyOrderOptimizable()));
@@ -395,12 +400,11 @@ public class HashJoinPlan extends DelegateQueryPlan {
}
@Override
- public void postProcess(Object result, HashJoinPlan parent)
+ public void postProcess(ServerCache result, HashJoinPlan parent)
throws SQLException {
- ServerCache cache = (ServerCache) result;
+ ServerCache cache = result;
if (cache != null) {
parent.joinInfo.getJoinIds()[index].set(cache.getId());
- parent.dependencies.add(cache);
}
}
[37/50] [abbrv] phoenix git commit: Revert "PHOENIX-2552 Fix hanging
Jenkins builds"
Posted by sa...@apache.org.
Revert "PHOENIX-2552 Fix hanging Jenkins builds"
This reverts commit b624fbf26615359683dfa5fd2b7919e758085d39.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/139e0d74
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/139e0d74
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/139e0d74
Branch: refs/heads/4.x-HBase-1.0
Commit: 139e0d74eb13cfa6b5da58a2ee5218c4a7cb08a5
Parents: dd3028c
Author: James Taylor <jt...@salesforce.com>
Authored: Sat Jan 2 10:29:50 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Sat Jan 2 10:29:50 2016 -0800
----------------------------------------------------------------------
.../main/java/org/apache/phoenix/memory/GlobalMemoryManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/139e0d74/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java b/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
index 18bc502..e70b35f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java
@@ -151,7 +151,7 @@ public class GlobalMemoryManager implements MemoryManager {
if (size > 0) {
logger.warn("Orphaned chunk of " + size + " bytes found during finalize");
}
- //freeMemory();
+ freeMemory();
} finally {
super.finalize();
}
[15/50] [abbrv] phoenix git commit: PHOENIX-2367 : Change Phoenix
Record Writer to use excecute instead of executeBatch()
Posted by sa...@apache.org.
PHOENIX-2367 : Change Phoenix Record Writer to use excecute instead of executeBatch()
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/785f4875
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/785f4875
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/785f4875
Branch: refs/heads/4.x-HBase-1.0
Commit: 785f487532169d1e66f4f4f6b03a97ce44c4432d
Parents: 9dce7c1
Author: Siddhi <sm...@salesforce.com>
Authored: Tue Dec 15 16:05:11 2015 -0800
Committer: Jan <jf...@salesforce.com>
Committed: Sun Dec 20 15:04:41 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/phoenix/mapreduce/PhoenixRecordWriter.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/785f4875/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordWriter.java
index 5843076..70ee3f5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordWriter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordWriter.java
@@ -55,7 +55,6 @@ public class PhoenixRecordWriter<T extends DBWritable> extends RecordWriter<Nul
@Override
public void close(TaskAttemptContext context) throws IOException, InterruptedException {
try {
- statement.executeBatch();
conn.commit();
} catch (SQLException e) {
LOG.error("SQLException while performing the commit for the task.");
@@ -77,10 +76,9 @@ public class PhoenixRecordWriter<T extends DBWritable> extends RecordWriter<Nul
try {
record.write(statement);
numRecords++;
- statement.addBatch();
+ statement.execute();
if (numRecords % batchSize == 0) {
LOG.debug("commit called on a batch of size : " + batchSize);
- statement.executeBatch();
conn.commit();
}
} catch (SQLException e) {
[26/50] [abbrv] phoenix git commit: PHOENIX-2545 Abort transaction if
send fails during commit
Posted by sa...@apache.org.
PHOENIX-2545 Abort transaction if send fails during commit
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/16e5c7bf
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/16e5c7bf
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/16e5c7bf
Branch: refs/heads/4.x-HBase-1.0
Commit: 16e5c7bf02a92c466b1730b6fa36371608d44f01
Parents: 55b7ada
Author: James Taylor <jt...@salesforce.com>
Authored: Sat Dec 26 23:47:31 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Dec 30 17:57:41 2015 -0800
----------------------------------------------------------------------
.../apache/phoenix/end2end/CreateTableIT.java | 28 +++
.../apache/phoenix/end2end/UpsertSelectIT.java | 2 +-
.../apache/phoenix/execute/PartialCommitIT.java | 44 ++--
.../coprocessor/MetaDataEndpointImpl.java | 13 +-
.../apache/phoenix/execute/MutationState.java | 216 ++++++++++++-------
.../index/PhoenixTransactionalIndexer.java | 17 +-
6 files changed, 207 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/16e5c7bf/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
index 7c4576c..5ffc354 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.query.KeyRange;
+import org.apache.phoenix.schema.NewerTableAlreadyExistsException;
import org.apache.phoenix.schema.TableAlreadyExistsException;
import org.apache.phoenix.util.PhoenixRuntime;
import org.junit.Test;
@@ -407,4 +408,31 @@ public class CreateTableIT extends BaseClientManagedTimeIT {
assertEquals(SQLExceptionCode.COLUMN_FAMILY_NOT_ALLOWED_FOR_TTL.getErrorCode(),sqle.getErrorCode());
}
}
+
+ @Test
+ public void testAlterDeletedTable() throws Exception {
+ String ddl = "create table T ("
+ + " K varchar primary key,"
+ + " V1 varchar)";
+ long ts = nextTimestamp();
+ Properties props = new Properties();
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts));
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement().execute(ddl);
+ conn.close();
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts+50));
+ Connection connAt50 = DriverManager.getConnection(getUrl(), props);
+ connAt50.createStatement().execute("DROP TABLE T");
+ connAt50.close();
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts+20));
+ Connection connAt20 = DriverManager.getConnection(getUrl(), props);
+ connAt20.createStatement().execute("UPDATE STATISTICS T"); // Invalidates from cache
+ try {
+ connAt20.createStatement().execute("ALTER TABLE T ADD V2 VARCHAR");
+ fail();
+ } catch (NewerTableAlreadyExistsException e) {
+
+ }
+ connAt20.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/16e5c7bf/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
index 689562a..b5252e0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
@@ -1109,7 +1109,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT {
}
// upsert data into base table without specifying the row timestamp column PK2
- long upsertedTs = 5;
+ long upsertedTs = nextTimestamp();
try (Connection conn = getConnection(upsertedTs)) {
// Upsert select in the same table with the row_timestamp column PK2 not specified. This will end up
// creating a new row whose timestamp is the SCN of the connection. The same SCN will be used
http://git-wip-us.apache.org/repos/asf/phoenix/blob/16e5c7bf/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java b/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
index 61aae62..8d7ebcb 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java
@@ -100,7 +100,10 @@ public class PartialCommitIT extends BaseOwnClusterIT {
return Arrays.asList(false, true);
}
+ private final boolean transactional;
+
public PartialCommitIT(boolean transactional) {
+ this.transactional = transactional;
if (transactional) {
A_SUCESS_TABLE = "A_SUCCESS_TABLE_TXN";
B_FAILURE_TABLE = TABLE_NAME_TO_FAIL+"_TXN";
@@ -148,8 +151,8 @@ public class PartialCommitIT extends BaseOwnClusterIT {
@Test
public void testNoFailure() {
- testPartialCommit(singletonList("upsert into " + A_SUCESS_TABLE + " values ('testNoFailure', 'a')"), 0, new int[0], false,
- singletonList("select count(*) from " + A_SUCESS_TABLE + " where k='testNoFailure'"), singletonList(new Integer(1)));
+ testPartialCommit(singletonList("upsert into " + A_SUCESS_TABLE + " values ('testNoFailure', 'a')"), new int[0], false, singletonList("select count(*) from " + A_SUCESS_TABLE + " where k='testNoFailure'"),
+ singletonList(new Integer(1)));
}
@Test
@@ -157,10 +160,10 @@ public class PartialCommitIT extends BaseOwnClusterIT {
testPartialCommit(newArrayList("upsert into " + A_SUCESS_TABLE + " values ('testUpsertFailure1', 'a')",
UPSERT_TO_FAIL,
"upsert into " + A_SUCESS_TABLE + " values ('testUpsertFailure2', 'b')"),
- 1, new int[]{1}, true,
+ transactional ? new int[] {0,1,2} : new int[]{1}, true,
newArrayList("select count(*) from " + A_SUCESS_TABLE + " where k like 'testUpsertFailure_'",
- "select count(*) from " + B_FAILURE_TABLE + " where k = '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "'"),
- newArrayList(new Integer(2), new Integer(0)));
+ "select count(*) from " + B_FAILURE_TABLE + " where k = '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "'"),
+ transactional ? newArrayList(new Integer(0), new Integer(0)) : newArrayList(new Integer(2), new Integer(0)));
}
@Test
@@ -172,10 +175,10 @@ public class PartialCommitIT extends BaseOwnClusterIT {
testPartialCommit(newArrayList("upsert into " + A_SUCESS_TABLE + " values ('testUpsertSelectFailure', 'a')",
UPSERT_SELECT_TO_FAIL),
- 1, new int[]{1}, true,
+ transactional ? new int[] {0,1} : new int[]{1}, true,
newArrayList("select count(*) from " + A_SUCESS_TABLE + " where k in ('testUpsertSelectFailure', '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "')",
- "select count(*) from " + B_FAILURE_TABLE + " where k = '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "'"),
- newArrayList(new Integer(2), new Integer(0)));
+ "select count(*) from " + B_FAILURE_TABLE + " where k = '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "'"),
+ transactional ? newArrayList(new Integer(1) /* from commit above */, new Integer(0)) : newArrayList(new Integer(2), new Integer(0)));
}
@Test
@@ -183,10 +186,10 @@ public class PartialCommitIT extends BaseOwnClusterIT {
testPartialCommit(newArrayList("upsert into " + A_SUCESS_TABLE + " values ('testDeleteFailure1', 'a')",
DELETE_TO_FAIL,
"upsert into " + A_SUCESS_TABLE + " values ('testDeleteFailure2', 'b')"),
- 1, new int[]{1}, true,
+ transactional ? new int[] {0,1,2} : new int[]{1}, true,
newArrayList("select count(*) from " + A_SUCESS_TABLE + " where k like 'testDeleteFailure_'",
- "select count(*) from " + B_FAILURE_TABLE + " where k = 'z'"),
- newArrayList(new Integer(2), new Integer(1)));
+ "select count(*) from " + B_FAILURE_TABLE + " where k = 'z'"),
+ transactional ? newArrayList(new Integer(0), new Integer(1) /* original row */) : newArrayList(new Integer(2), new Integer(1)));
}
/**
@@ -197,11 +200,11 @@ public class PartialCommitIT extends BaseOwnClusterIT {
testPartialCommit(newArrayList("upsert into " + C_SUCESS_TABLE + " values ('testOrderOfMutationsIsPredicatable', 'c')", // will fail because c_success_table is after b_failure_table by table sort order
UPSERT_TO_FAIL,
"upsert into " + A_SUCESS_TABLE + " values ('testOrderOfMutationsIsPredicatable', 'a')"), // will succeed because a_success_table is before b_failure_table by table sort order
- 2, new int[]{0,1}, true,
+ transactional ? new int[] {0,1,2} : new int[]{0,1}, true,
newArrayList("select count(*) from " + C_SUCESS_TABLE + " where k='testOrderOfMutationsIsPredicatable'",
"select count(*) from " + A_SUCESS_TABLE + " where k='testOrderOfMutationsIsPredicatable'",
- "select count(*) from " + B_FAILURE_TABLE + " where k = '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "'"),
- newArrayList(new Integer(0), new Integer(1), new Integer(0)));
+ "select count(*) from " + B_FAILURE_TABLE + " where k = '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "'"),
+ transactional ? newArrayList(new Integer(0), new Integer(0), new Integer(0)) : newArrayList(new Integer(0), new Integer(1), new Integer(0)));
}
@Test
@@ -211,15 +214,15 @@ public class PartialCommitIT extends BaseOwnClusterIT {
DELETE_TO_FAIL,
"select * from " + A_SUCESS_TABLE + "",
UPSERT_TO_FAIL),
- 2, new int[]{2,4}, true,
+ transactional ? new int[] {0,1,2,4} : new int[]{2,4}, true,
newArrayList("select count(*) from " + A_SUCESS_TABLE + " where k='testStatementOrderMaintainedInConnection' or k like 'z%'", // rows left: zz, zzz, checkThatAllStatementTypesMaintainOrderInConnection
"select count(*) from " + B_FAILURE_TABLE + " where k = '" + Bytes.toString(ROW_TO_FAIL_UPSERT_BYTES) + "'",
- "select count(*) from " + B_FAILURE_TABLE + " where k = 'z'"),
- newArrayList(new Integer(4), new Integer(0), new Integer(1)));
+ "select count(*) from " + B_FAILURE_TABLE + " where k = 'z'"),
+ transactional ? newArrayList(new Integer(3) /* original rows */, new Integer(0), new Integer(1) /* original row */) : newArrayList(new Integer(4), new Integer(0), new Integer(1)));
}
- private void testPartialCommit(List<String> statements, int failureCount, int[] expectedUncommittedStatementIndexes, boolean willFail,
- List<String> countStatementsForVerification, List<Integer> expectedCountsForVerification) {
+ private void testPartialCommit(List<String> statements, int[] expectedUncommittedStatementIndexes, boolean willFail, List<String> countStatementsForVerification,
+ List<Integer> expectedCountsForVerification) {
Preconditions.checkArgument(countStatementsForVerification.size() == expectedCountsForVerification.size());
try (Connection con = getConnectionWithTableOrderPreservingMutationState()) {
@@ -241,7 +244,6 @@ public class PartialCommitIT extends BaseOwnClusterIT {
}
assertEquals(CommitException.class, sqle.getClass());
int[] uncommittedStatementIndexes = ((CommitException)sqle).getUncommittedStatementIndexes();
- assertEquals(failureCount, uncommittedStatementIndexes.length);
assertArrayEquals(expectedUncommittedStatementIndexes, uncommittedStatementIndexes);
}
@@ -267,7 +269,7 @@ public class PartialCommitIT extends BaseOwnClusterIT {
return new PhoenixConnection(phxCon, null) {
@Override
protected MutationState newMutationState(int maxSize) {
- return new MutationState(maxSize, this, mutations, null);
+ return new MutationState(maxSize, this, mutations, null, null);
};
};
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/16e5c7bf/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 2043c66..ea4a7e1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -978,13 +978,12 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
scan.setFilter(new FirstKeyOnlyFilter());
scan.setRaw(true);
List<Cell> results = Lists.<Cell> newArrayList();
- try (RegionScanner scanner = region.getScanner(scan);) {
+ try (RegionScanner scanner = region.getScanner(scan)) {
scanner.next(results);
}
- // HBase ignores the time range on a raw scan (HBASE-7362)
- if (!results.isEmpty() && results.get(0).getTimestamp() > clientTimeStamp) {
- Cell kv = results.get(0);
- if (kv.getTypeByte() == Type.Delete.getCode()) {
+ for (Cell kv : results) {
+ KeyValue.Type type = Type.codeToType(kv.getTypeByte());
+ if (type == Type.DeleteFamily) { // Row was deleted
Cache<ImmutableBytesPtr, PMetaDataEntity> metaDataCache =
GlobalCache.getInstance(this.env).getMetaDataCache();
PTable table = newDeletedTableMarker(kv.getTimestamp());
@@ -1621,7 +1620,9 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
&& (table = buildTable(key, cacheKey, region, HConstants.LATEST_TIMESTAMP)) == null) {
// if not found then call newerTableExists and add delete marker for timestamp
// found
- if (buildDeletedTable(key, cacheKey, region, clientTimeStamp) != null) {
+ table = buildDeletedTable(key, cacheKey, region, clientTimeStamp);
+ if (table != null) {
+ logger.info("Found newer table deleted as of " + table.getTimeStamp());
return new MetaDataMutationResult(MutationCode.NEWER_TABLE_FOUND,
EnvironmentEdgeManager.currentTimeMillis(), null);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/16e5c7bf/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
index 52bbeb6..fa5e962 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -25,6 +25,7 @@ import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_MUTATION_
import java.io.IOException;
import java.sql.SQLException;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -70,6 +71,7 @@ import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.RowKeySchema;
+import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.ValueSchema.Field;
import org.apache.phoenix.schema.types.PLong;
@@ -109,8 +111,9 @@ import co.cask.tephra.hbase10.TransactionAwareHTable;
public class MutationState implements SQLCloseable {
private static final Logger logger = LoggerFactory.getLogger(MutationState.class);
private static final TransactionCodec CODEC = new TransactionCodec();
+ private static final int[] EMPTY_STATEMENT_INDEX_ARRAY = new int[0];
- private PhoenixConnection connection;
+ private final PhoenixConnection connection;
private final long maxSize;
private final Map<TableRef, Map<ImmutableBytesPtr,RowMutationState>> mutations;
private final List<TransactionAware> txAwares;
@@ -120,35 +123,39 @@ public class MutationState implements SQLCloseable {
private Transaction tx;
private long sizeOffset;
private int numRows = 0;
- private boolean txStarted = false;
+ private int[] uncommittedStatementIndexes = EMPTY_STATEMENT_INDEX_ARRAY;
private final MutationMetricQueue mutationMetricQueue;
private ReadMetricQueue readMetricQueue;
public MutationState(long maxSize, PhoenixConnection connection) {
- this(maxSize,connection, null);
+ this(maxSize,connection, null, null);
+ }
+
+ public MutationState(long maxSize, PhoenixConnection connection, TransactionContext txContext) {
+ this(maxSize,connection, null, txContext);
}
public MutationState(MutationState mutationState) {
- this(mutationState.maxSize, mutationState.connection, mutationState.getTransaction());
+ this(mutationState.maxSize, mutationState.connection, mutationState.getTransaction(), null);
}
public MutationState(long maxSize, PhoenixConnection connection, long sizeOffset) {
- this(maxSize, connection, null, sizeOffset);
+ this(maxSize, connection, null, null, sizeOffset);
}
- private MutationState(long maxSize, PhoenixConnection connection, Transaction tx) {
- this(maxSize,connection, tx, 0);
+ private MutationState(long maxSize, PhoenixConnection connection, Transaction tx, TransactionContext txContext) {
+ this(maxSize,connection, tx, txContext, 0);
}
- private MutationState(long maxSize, PhoenixConnection connection, Transaction tx, long sizeOffset) {
- this(maxSize, connection, Maps.<TableRef, Map<ImmutableBytesPtr,RowMutationState>>newHashMapWithExpectedSize(connection.getMutateBatchSize()), tx);
+ private MutationState(long maxSize, PhoenixConnection connection, Transaction tx, TransactionContext txContext, long sizeOffset) {
+ this(maxSize, connection, Maps.<TableRef, Map<ImmutableBytesPtr,RowMutationState>>newHashMapWithExpectedSize(connection.getMutateBatchSize()), tx, txContext);
this.sizeOffset = sizeOffset;
}
MutationState(long maxSize, PhoenixConnection connection,
Map<TableRef, Map<ImmutableBytesPtr, RowMutationState>> mutations,
- Transaction tx) {
+ Transaction tx, TransactionContext txContext) {
this.maxSize = maxSize;
this.connection = connection;
this.mutations = mutations;
@@ -157,26 +164,34 @@ public class MutationState implements SQLCloseable {
: NoOpMutationMetricsQueue.NO_OP_MUTATION_METRICS_QUEUE;
this.tx = tx;
if (tx == null) {
- this.txAwares = Collections.emptyList();
- TransactionSystemClient txServiceClient = this.connection
- .getQueryServices().getTransactionSystemClient();
- this.txContext = new TransactionContext(txServiceClient);
+ this.txAwares = Collections.emptyList();
+ if (txContext == null) {
+ TransactionSystemClient txServiceClient = this.connection
+ .getQueryServices().getTransactionSystemClient();
+ this.txContext = new TransactionContext(txServiceClient);
+ } else {
+ this.txContext = txContext;
+ }
} else {
// this code path is only used while running child scans, we can't pass the txContext to child scans
// as it is not thread safe, so we use the tx member variable
- txAwares = Lists.newArrayList();
- txContext = null;
+ this.txAwares = Lists.newArrayList();
+ this.txContext = null;
}
}
public MutationState(TableRef table, Map<ImmutableBytesPtr,RowMutationState> mutations, long sizeOffset, long maxSize, PhoenixConnection connection) {
- this(maxSize, connection, null, sizeOffset);
+ this(maxSize, connection, null, null, sizeOffset);
this.mutations.put(table, mutations);
this.numRows = mutations.size();
this.tx = connection.getMutationState().getTransaction();
throwIfTooBig();
}
+ public long getMaxSize() {
+ return maxSize;
+ }
+
public boolean checkpointIfNeccessary(MutationPlan plan) throws SQLException {
Transaction currentTx = getTransaction();
if (getTransaction() == null || plan.getTargetRef() == null || plan.getTargetRef().getTable() == null || !plan.getTargetRef().getTable().isTransactional()) {
@@ -308,9 +323,8 @@ public class MutationState implements SQLCloseable {
}
try {
- if (!txStarted) {
+ if (!isTransactionStarted()) {
txContext.start();
- txStarted = true;
return true;
}
} catch (TransactionFailureException e) {
@@ -320,7 +334,7 @@ public class MutationState implements SQLCloseable {
}
public static MutationState emptyMutationState(long maxSize, PhoenixConnection connection) {
- MutationState state = new MutationState(maxSize, connection, Collections.<TableRef, Map<ImmutableBytesPtr,RowMutationState>>emptyMap(), null);
+ MutationState state = new MutationState(maxSize, connection, Collections.<TableRef, Map<ImmutableBytesPtr,RowMutationState>>emptyMap(), null, null);
state.sizeOffset = 0;
return state;
}
@@ -599,18 +613,24 @@ public class MutationState implements SQLCloseable {
Long scn = connection.getSCN();
MetaDataClient client = new MetaDataClient(connection);
long serverTimeStamp = tableRef.getTimeStamp();
- PTable table = tableRef.getTable();
// If we're auto committing, we've already validated the schema when we got the ColumnResolver,
// so no need to do it again here.
if (!connection.getAutoCommit()) {
+ PTable table = tableRef.getTable();
MetaDataMutationResult result = client.updateCache(table.getSchemaName().getString(), table.getTableName().getString());
+ PTable resolvedTable = result.getTable();
+ if (resolvedTable == null) {
+ throw new TableNotFoundException(table.getSchemaName().getString(), table.getTableName().getString());
+ }
+ // Always update tableRef table as the one we've cached may be out of date since when we executed
+ // the UPSERT VALUES call and updated in the cache before this.
+ tableRef.setTable(resolvedTable);
long timestamp = result.getMutationTime();
if (timestamp != QueryConstants.UNSET_TIMESTAMP) {
serverTimeStamp = timestamp;
if (result.wasUpdated()) {
// TODO: use bitset?
- table = result.getTable();
- PColumn[] columns = new PColumn[table.getColumns().size()];
+ PColumn[] columns = new PColumn[resolvedTable.getColumns().size()];
for (Map.Entry<ImmutableBytesPtr,RowMutationState> rowEntry : rowKeyToColumnMap.entrySet()) {
RowMutationState valueEntry = rowEntry.getValue();
if (valueEntry != null) {
@@ -624,10 +644,9 @@ public class MutationState implements SQLCloseable {
}
for (PColumn column : columns) {
if (column != null) {
- table.getColumnFamily(column.getFamilyName().getString()).getColumn(column.getName().getString());
+ resolvedTable.getColumnFamily(column.getFamilyName().getString()).getColumn(column.getName().getString());
}
}
- tableRef.setTable(table);
}
}
}
@@ -731,25 +750,28 @@ public class MutationState implements SQLCloseable {
sendAll = true;
}
+ List<TableRef> txTableRefs = Lists.newArrayListWithExpectedSize(mutations.size());
// add tracing for this operation
try (TraceScope trace = Tracing.startNewSpan(connection, "Committing mutations to tables")) {
Span span = trace.getSpan();
ImmutableBytesWritable indexMetaDataPtr = new ImmutableBytesWritable();
+ boolean isTransactional;
while (tableRefIterator.hasNext()) {
// at this point we are going through mutations for each table
- TableRef tableRef = tableRefIterator.next();
+ final TableRef tableRef = tableRefIterator.next();
Map<ImmutableBytesPtr, RowMutationState> valuesMap = mutations.get(tableRef);
if (valuesMap == null || valuesMap.isEmpty()) {
continue;
}
- PTable table = tableRef.getTable();
+ // Validate as we go if transactional since we can undo if a problem occurs (which is unlikely)
+ long serverTimestamp = serverTimeStamps == null ? validate(tableRef, valuesMap) : serverTimeStamps[i++];
+ final PTable table = tableRef.getTable();
// Track tables to which we've sent uncommitted data
- if (table.isTransactional()) {
+ if (isTransactional = table.isTransactional()) {
+ txTableRefs.add(tableRef);
uncommittedPhysicalNames.add(table.getPhysicalName().getString());
}
boolean isDataTable = true;
- // Validate as we go if transactional since we can undo if a problem occurs (which is unlikely)
- long serverTimestamp = serverTimeStamps == null ? validate(tableRef, valuesMap) : serverTimeStamps[i++];
table.getIndexMaintainers(indexMetaDataPtr, connection);
Iterator<Pair<byte[],List<Mutation>>> mutationsIterator = addRowMutations(tableRef, valuesMap, serverTimestamp, false, sendAll);
while (mutationsIterator.hasNext()) {
@@ -776,7 +798,7 @@ public class MutationState implements SQLCloseable {
SQLException sqlE = null;
HTableInterface hTable = connection.getQueryServices().getTable(htableName);
try {
- if (table.isTransactional()) {
+ if (isTransactional) {
// If we have indexes, wrap the HTable in a delegate HTable that
// will attach the necessary index meta data in the event of a
// rollback
@@ -830,8 +852,8 @@ public class MutationState implements SQLCloseable {
}
e = inferredE;
}
- // Throw to client with both what was committed so far and what is left to be committed.
- // That way, client can either undo what was done or try again with what was not done.
+ // Throw to client an exception that indicates the statements that
+ // were not committed successfully.
sqlE = new CommitException(e, getUncommittedStatementIndexes());
} finally {
try {
@@ -862,17 +884,24 @@ public class MutationState implements SQLCloseable {
if (tableRef.getTable().getType() != PTableType.INDEX) {
numRows -= valuesMap.size();
}
- // Remove batches as we process them
+ // For transactions, track the statement indexes as we send data
+ // over because our CommitException should include all statements
+ // involved in the transaction since none of them would have been
+ // committed in the event of a failure.
+ if (isTransactional) {
+ addUncommittedStatementIndexes(valuesMap.values());
+ }
+ // Remove batches as we process them
if (sendAll) {
- tableRefIterator.remove(); // Iterating through actual map in this case
+ // Iterating through map key set in this case, so we cannot use
+ // the remove method without getting a concurrent modification
+ // exception.
+ tableRefIterator.remove();
} else {
mutations.remove(tableRef);
}
}
}
- // Note that we cannot assume that *all* mutations have been sent, since we've optimized this
- // now to only send the mutations for the tables we're querying, hence we've removed the
- // assertions that we're here before.
}
public byte[] encodeTransaction() throws SQLException {
@@ -930,19 +959,17 @@ public class MutationState implements SQLCloseable {
return cache;
}
- private void clear() throws SQLException {
- this.mutations.clear();
- numRows = 0;
+ private void addUncommittedStatementIndexes(Collection<RowMutationState> rowMutations) {
+ for (RowMutationState rowMutationState : rowMutations) {
+ uncommittedStatementIndexes = joinSortedIntArrays(uncommittedStatementIndexes, rowMutationState.getStatementIndexes());
+ }
}
private int[] getUncommittedStatementIndexes() {
- int[] result = new int[0];
- for (Map<ImmutableBytesPtr, RowMutationState> rowMutations : mutations.values()) {
- for (RowMutationState rowMutationState : rowMutations.values()) {
- result = joinSortedIntArrays(result, rowMutationState.getStatementIndexes());
- }
+ for (Map<ImmutableBytesPtr, RowMutationState> rowMutationMap : mutations.values()) {
+ addUncommittedStatementIndexes(rowMutationMap.values());
}
- return result;
+ return uncommittedStatementIndexes;
}
@Override
@@ -950,65 +977,96 @@ public class MutationState implements SQLCloseable {
}
private void reset() {
- txStarted = false;
tx = null;
+ txAwares.clear();
uncommittedPhysicalNames.clear();
+ this.mutations.clear();
+ numRows = 0;
+ uncommittedStatementIndexes = EMPTY_STATEMENT_INDEX_ARRAY;
}
public void rollback() throws SQLException {
- clear();
- txAwares.clear();
- if (txContext != null) {
- try {
- if (txStarted) {
+ try {
+ if (txContext != null && isTransactionStarted()) {
+ try {
txContext.abort();
+ } catch (TransactionFailureException e) {
+ throw TransactionUtil.getTransactionFailureException(e);
}
- } catch (TransactionFailureException e) {
- throw new SQLException(e); // TODO: error code
- } finally {
- reset();
}
+ } finally {
+ reset();
}
}
public void commit() throws SQLException {
- boolean sendMutationsFailed=false;
+ boolean sendSuccessful=false;
+ SQLException sqlE = null;
try {
send();
- } catch (Throwable t) {
- sendMutationsFailed=true;
- throw t;
+ sendSuccessful=true;
+ } catch (SQLException e) {
+ sqlE = e;
} finally {
- txAwares.clear();
- if (txContext != null) {
- try {
- if (txStarted && !sendMutationsFailed) {
- txContext.finish();
- }
- } catch (TransactionFailureException e) {
+ try {
+ if (txContext != null && isTransactionStarted()) {
+ TransactionFailureException txFailure = null;
+ boolean finishSuccessful=false;
try {
- txContext.abort(e);
- // abort and throw the original commit failure exception
- throw TransactionUtil.getTransactionFailureException(e);
- } catch (TransactionFailureException e1) {
- // if abort fails and throw the abort failure exception
- throw TransactionUtil.getTransactionFailureException(e1);
+ if (sendSuccessful) {
+ txContext.finish();
+ finishSuccessful = true;
+ }
+ } catch (TransactionFailureException e) {
+ txFailure = e;
+ SQLException nextE = TransactionUtil.getTransactionFailureException(e);
+ if (sqlE == null) {
+ sqlE = nextE;
+ } else {
+ sqlE.setNextException(nextE);
+ }
+ } finally {
+ // If send fails or finish fails, abort the tx
+ if (!finishSuccessful) {
+ try {
+ txContext.abort(txFailure);
+ } catch (TransactionFailureException e) {
+ SQLException nextE = TransactionUtil.getTransactionFailureException(e);
+ if (sqlE == null) {
+ sqlE = nextE;
+ } else {
+ sqlE.setNextException(nextE);
+ }
+ }
+ }
}
+ }
+ } finally {
+ try {
+ reset();
} finally {
- if (!sendMutationsFailed) {
- reset();
- }
- }
+ if (sqlE != null) {
+ throw sqlE;
+ }
+ }
}
}
}
/**
+ * Send to HBase any uncommitted data for transactional tables.
+ * @return true if any data was sent and false otherwise.
+ * @throws SQLException
+ */
+ public boolean sendUncommitted() throws SQLException {
+ return sendUncommitted(mutations.keySet().iterator());
+ }
+ /**
* Support read-your-own-write semantics by sending uncommitted data to HBase prior to running a
* query. In this way, they are visible to subsequent reads but are not actually committed until
* commit is called.
* @param tableRefs
- * @return true if at least partially transactional and false otherwise.
+ * @return true if any data was sent and false otherwise.
* @throws SQLException
*/
public boolean sendUncommitted(Iterator<TableRef> tableRefs) throws SQLException {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/16e5c7bf/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
index 3fecc18..2925b09 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
@@ -71,16 +71,19 @@ import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.ServerUtil;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.Transaction.VisibilityLevel;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.hbase10.TransactionAwareHTable;
-
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.primitives.Longs;
+import co.cask.tephra.Transaction;
+import co.cask.tephra.Transaction;
+import co.cask.tephra.Transaction.VisibilityLevel;
+import co.cask.tephra.Transaction.VisibilityLevel;
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.hbase10.TransactionAwareHTable;
+
/**
* Do all the work of managing index updates for a transactional table from a single coprocessor. Since the transaction
* manager essentially time orders writes through conflict detection, the logic to maintain a secondary index is quite a
@@ -196,7 +199,9 @@ public class PhoenixTransactionalIndexer extends BaseRegionObserver {
// run a scan to get the current state. We'll need this to delete
// the existing index rows.
Transaction tx = indexMetaData.getTransaction();
- assert(tx != null);
+ if (tx == null) {
+ throw new NullPointerException("Expected to find transaction in metadata for " + env.getRegionInfo().getTable().getNameAsString());
+ }
List<IndexMaintainer> indexMaintainers = indexMetaData.getIndexMaintainers();
Set<ColumnReference> mutableColumns = Sets.newHashSetWithExpectedSize(indexMaintainers.size() * 10);
for (IndexMaintainer indexMaintainer : indexMaintainers) {
[27/50] [abbrv] phoenix git commit: PHOENIX-2411 Allow Phoenix to
participate as transactional component
Posted by sa...@apache.org.
PHOENIX-2411 Allow Phoenix to participate as transactional component
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/cd4f4547
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/cd4f4547
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/cd4f4547
Branch: refs/heads/4.x-HBase-1.0
Commit: cd4f4547b77d3da9816fc9cbfad59298a04211ca
Parents: 16e5c7b
Author: James Taylor <jt...@salesforce.com>
Authored: Sat Dec 26 23:56:59 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Dec 30 18:14:20 2015 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/tx/TransactionIT.java | 136 ++++++++++++++++++-
.../phoenix/exception/SQLExceptionCode.java | 2 +
.../apache/phoenix/execute/MutationState.java | 9 +-
.../apache/phoenix/jdbc/PhoenixConnection.java | 58 ++++++--
.../apache/phoenix/jdbc/PhoenixStatement.java | 13 +-
.../org/apache/phoenix/query/QueryServices.java | 9 +-
.../phoenix/query/QueryServicesOptions.java | 6 +-
.../org/apache/phoenix/util/PhoenixRuntime.java | 62 ++++++---
8 files changed, 249 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cd4f4547/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
index 929e4a5..e83467a 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
@@ -20,8 +20,6 @@ package org.apache.phoenix.tx;
import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.apache.phoenix.util.TestUtil.TRANSACTIONAL_DATA_TABLE;
-import static org.apache.phoenix.util.TestUtil.analyzeTable;
-import static org.apache.phoenix.util.TestUtil.getAllSplits;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -41,22 +39,22 @@ import java.util.Properties;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
import org.apache.phoenix.end2end.Shadower;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
-import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableKey;
-import org.apache.phoenix.schema.ReadOnlyTableException;
import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.PropertiesUtil;
@@ -66,12 +64,15 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.hbase10.coprocessor.TransactionProcessor;
-
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import co.cask.tephra.TransactionContext;
+import co.cask.tephra.TransactionSystemClient;
+import co.cask.tephra.TxConstants;
+import co.cask.tephra.hbase10.TransactionAwareHTable;
+import co.cask.tephra.hbase10.coprocessor.TransactionProcessor;
+
public class TransactionIT extends BaseHBaseManagedTimeIT {
private static final String FULL_TABLE_NAME = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + TRANSACTIONAL_DATA_TABLE;
@@ -603,4 +604,125 @@ public class TransactionIT extends BaseHBaseManagedTimeIT {
}
assertEquals(5, count);
}
+
+ @Test
+ public void testExternalTxContext() throws Exception {
+ ResultSet rs;
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+
+ TransactionSystemClient txServiceClient = pconn.getQueryServices().getTransactionSystemClient();
+
+ String fullTableName = "T";
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE " + fullTableName + "(K VARCHAR PRIMARY KEY, V1 VARCHAR, V2 VARCHAR) TRANSACTIONAL=true");
+ HTableInterface htable = pconn.getQueryServices().getTable(Bytes.toBytes(fullTableName));
+ stmt.executeUpdate("upsert into " + fullTableName + " values('x', 'a', 'a')");
+ conn.commit();
+
+ try (Connection newConn = DriverManager.getConnection(getUrl(), props)) {
+ rs = newConn.createStatement().executeQuery("select count(*) from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals(1,rs.getInt(1));
+ }
+
+ // Use HBase level Tephra APIs to start a new transaction
+ TransactionAwareHTable txAware = new TransactionAwareHTable(htable, TxConstants.ConflictDetection.ROW);
+ TransactionContext txContext = new TransactionContext(txServiceClient, txAware);
+ txContext.start();
+
+ // Use HBase APIs to add a new row
+ Put put = new Put(Bytes.toBytes("z"));
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES);
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, Bytes.toBytes("V1"), Bytes.toBytes("b"));
+ txAware.put(put);
+
+ // Use Phoenix APIs to add new row (sharing the transaction context)
+ pconn.setTransactionContext(txContext);
+ conn.createStatement().executeUpdate("upsert into " + fullTableName + " values('y', 'c', 'c')");
+
+ // New connection should not see data as it hasn't been committed yet
+ try (Connection newConn = DriverManager.getConnection(getUrl(), props)) {
+ rs = newConn.createStatement().executeQuery("select count(*) from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals(1,rs.getInt(1));
+ }
+
+ // Use new connection to create a row with a conflict
+ Connection connWithConflict = DriverManager.getConnection(getUrl(), props);
+ connWithConflict.createStatement().execute("upsert into " + fullTableName + " values('z', 'd', 'd')");
+
+ // Existing connection should see data even though it hasn't been committed yet
+ rs = conn.createStatement().executeQuery("select count(*) from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals(3,rs.getInt(1));
+
+ // Use Tephra APIs directly to finish (i.e. commit) the transaction
+ txContext.finish();
+
+ // Confirm that attempt to commit row with conflict fails
+ try {
+ connWithConflict.commit();
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.TRANSACTION_CONFLICT_EXCEPTION.getErrorCode(), e.getErrorCode());
+ } finally {
+ connWithConflict.close();
+ }
+
+ // New connection should now see data as it has been committed
+ try (Connection newConn = DriverManager.getConnection(getUrl(), props)) {
+ rs = newConn.createStatement().executeQuery("select count(*) from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals(3,rs.getInt(1));
+ }
+
+ // Repeat the same as above, but this time abort the transaction
+ txContext = new TransactionContext(txServiceClient, txAware);
+ txContext.start();
+
+ // Use HBase APIs to add a new row
+ put = new Put(Bytes.toBytes("j"));
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES);
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, Bytes.toBytes("V1"), Bytes.toBytes("e"));
+ txAware.put(put);
+
+ // Use Phoenix APIs to add new row (sharing the transaction context)
+ pconn.setTransactionContext(txContext);
+ conn.createStatement().executeUpdate("upsert into " + fullTableName + " values('k', 'f', 'f')");
+
+ // Existing connection should see data even though it hasn't been committed yet
+ rs = conn.createStatement().executeQuery("select count(*) from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals(5,rs.getInt(1));
+
+ connWithConflict.createStatement().execute("upsert into " + fullTableName + " values('k', 'g', 'g')");
+ rs = connWithConflict.createStatement().executeQuery("select count(*) from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals(4,rs.getInt(1));
+
+ // Use Tephra APIs directly to abort (i.e. rollback) the transaction
+ txContext.abort();
+
+ rs = conn.createStatement().executeQuery("select count(*) from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals(3,rs.getInt(1));
+
+ // Should succeed since conflicting row was aborted
+ connWithConflict.commit();
+
+ // New connection should now see data as it has been committed
+ try (Connection newConn = DriverManager.getConnection(getUrl(), props)) {
+ rs = newConn.createStatement().executeQuery("select count(*) from " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals(4,rs.getInt(1));
+ }
+
+ // Even using HBase APIs directly, we shouldn't find 'j' since a delete marker would have been
+ // written to hide it.
+ Result result = htable.get(new Get(Bytes.toBytes("j")));
+ assertTrue(result.isEmpty());
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cd4f4547/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index ef135eb..36036ae 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -280,6 +280,8 @@ public enum SQLExceptionCode {
CANNOT_ALTER_TO_BE_TXN_IF_TXNS_DISABLED(1079, "44A10", "Cannot alter table to be transactional table if transactions are disabled"),
CANNOT_CREATE_TXN_TABLE_WITH_ROW_TIMESTAMP(1080, "44A11", "Cannot create a transactional table if transactions are disabled"),
CANNOT_ALTER_TO_BE_TXN_WITH_ROW_TIMESTAMP(1081, "44A12", "Cannot alter table to be transactional table if transactions are disabled"),
+ TX_MUST_BE_ENABLED_TO_SET_TX_CONTEXT(1082, "44A13", "Cannot set transaction context if transactions are disabled"),
+ TX_MUST_BE_ENABLED_TO_SET_AUTO_FLUSH(1083, "44A14", "Cannot set auto flush if transactions are disabled"),
/** Sequence related */
SEQUENCE_ALREADY_EXIST(1200, "42Z00", "Sequence already exists.", new Factory() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cd4f4547/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
index fa5e962..8ae9481 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -124,6 +124,7 @@ public class MutationState implements SQLCloseable {
private long sizeOffset;
private int numRows = 0;
private int[] uncommittedStatementIndexes = EMPTY_STATEMENT_INDEX_ARRAY;
+ private boolean isExternalTxContext = false;
private final MutationMetricQueue mutationMetricQueue;
private ReadMetricQueue readMetricQueue;
@@ -170,6 +171,7 @@ public class MutationState implements SQLCloseable {
.getQueryServices().getTransactionSystemClient();
this.txContext = new TransactionContext(txServiceClient);
} else {
+ isExternalTxContext = true;
this.txContext = txContext;
}
} else {
@@ -224,7 +226,12 @@ public class MutationState implements SQLCloseable {
boolean hasUncommittedData = false;
for (TableRef source : sources) {
String sourcePhysicalName = source.getTable().getPhysicalName().getString();
- if (source.getTable().isTransactional() && uncommittedPhysicalNames.contains(sourcePhysicalName)) {
+ // Tracking uncommitted physical table names is an optimization that prevents us from
+ // having to do a checkpoint if no data has yet been written. If we're using an
+ // external transaction context, it's possible that data was already written at the
+ // current transaction timestamp, so we always checkpoint in that case is we're
+ // reading and writing to the same table.
+ if (source.getTable().isTransactional() && (isExternalTxContext || uncommittedPhysicalNames.contains(sourcePhysicalName))) {
hasUncommittedData = true;
break;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cd4f4547/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
index 9ef1643..d33f20c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
@@ -107,6 +107,8 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMap.Builder;
import com.google.common.collect.Lists;
+import co.cask.tephra.TransactionContext;
+
/**
*
@@ -125,11 +127,12 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
private final String url;
private final ConnectionQueryServices services;
private final Properties info;
- private List<SQLCloseable> statements = new ArrayList<SQLCloseable>();
private final Map<PDataType<?>, Format> formatters = new HashMap<>();
- private final MutationState mutationState;
private final int mutateBatchSize;
private final Long scn;
+ private MutationState mutationState;
+ private List<SQLCloseable> statements = new ArrayList<SQLCloseable>();
+ private boolean isAutoFlush = false;
private boolean isAutoCommit = false;
private PMetaData metaData;
private final PName tenantId;
@@ -160,6 +163,7 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
public PhoenixConnection(PhoenixConnection connection, boolean isDescRowKeyOrderUpgrade) throws SQLException {
this(connection.getQueryServices(), connection.getURL(), connection.getClientInfo(), connection.metaData, connection.getMutationState(), isDescRowKeyOrderUpgrade);
this.isAutoCommit = connection.isAutoCommit;
+ this.isAutoFlush = connection.isAutoFlush;
this.sampler = connection.sampler;
this.statementExecutionCounter = connection.statementExecutionCounter;
}
@@ -179,6 +183,7 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
public PhoenixConnection(ConnectionQueryServices services, PhoenixConnection connection, long scn) throws SQLException {
this(services, connection.getURL(), newPropsWithSCN(scn,connection.getClientInfo()), connection.metaData, connection.getMutationState(), connection.isDescVarLengthRowKeyUpgrade());
this.isAutoCommit = connection.isAutoCommit;
+ this.isAutoFlush = connection.isAutoFlush;
this.sampler = connection.sampler;
this.statementExecutionCounter = connection.statementExecutionCounter;
}
@@ -218,6 +223,8 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
Long scnParam = JDBCUtil.getCurrentSCN(url, this.info);
checkScn(scnParam);
this.scn = scnParam;
+ this.isAutoFlush = this.services.getProps().getBoolean(QueryServices.TRANSACTIONS_ENABLED, QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED)
+ && this.services.getProps().getBoolean(QueryServices.AUTO_FLUSH_ATTRIB, QueryServicesOptions.DEFAULT_AUTO_FLUSH) ;
this.isAutoCommit = JDBCUtil.getAutoCommit(
url, this.info,
this.services.getProps().getBoolean(
@@ -281,17 +288,13 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
private static Properties filterKnownNonProperties(Properties info) {
Properties prunedProperties = info;
- if (info.contains(PhoenixRuntime.CURRENT_SCN_ATTRIB)) {
- if (prunedProperties == info) {
- prunedProperties = PropertiesUtil.deepCopy(info);
- }
- prunedProperties.remove(PhoenixRuntime.CURRENT_SCN_ATTRIB);
- }
- if (info.contains(PhoenixRuntime.TENANT_ID_ATTRIB)) {
- if (prunedProperties == info) {
- prunedProperties = PropertiesUtil.deepCopy(info);
+ for (String property : PhoenixRuntime.CONNECTION_PROPERTIES) {
+ if (info.contains(property)) {
+ if (prunedProperties == info) {
+ prunedProperties = PropertiesUtil.deepCopy(info);
+ }
+ prunedProperties.remove(property);
}
- prunedProperties.remove(PhoenixRuntime.TENANT_ID_ATTRIB);
}
return prunedProperties;
}
@@ -578,11 +581,40 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd
return isAutoCommit;
}
+ public boolean getAutoFlush() {
+ return isAutoFlush;
+ }
+
+ public void setAutoFlush(boolean autoFlush) throws SQLException {
+ if (autoFlush && !this.services.getProps().getBoolean(QueryServices.TRANSACTIONS_ENABLED, QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED)) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.TX_MUST_BE_ENABLED_TO_SET_AUTO_FLUSH)
+ .build().buildException();
+ }
+ this.isAutoFlush = autoFlush;
+ }
+
+ public void flush() throws SQLException {
+ mutationState.sendUncommitted();
+ }
+
+ public void setTransactionContext(TransactionContext txContext) throws SQLException {
+ if (!this.services.getProps().getBoolean(QueryServices.TRANSACTIONS_ENABLED, QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED)) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.TX_MUST_BE_ENABLED_TO_SET_TX_CONTEXT)
+ .build().buildException();
+ }
+ this.mutationState.rollback();
+ this.mutationState = new MutationState(this.mutationState.getMaxSize(), this, txContext);
+
+ // Write data to HBase after each statement execution as the commit may not
+ // come through Phoenix APIs.
+ setAutoFlush(true);
+ }
+
public Consistency getConsistency() {
return this.consistency;
}
- @Override
+ @Override
public String getCatalog() throws SQLException {
return "";
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cd4f4547/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index bacea92..fe62ea4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -1217,6 +1217,8 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho
PhoenixPreparedStatement statement = batch.get(i);
returnCodes[i] = statement.execute(true) ? Statement.SUCCESS_NO_INFO : statement.getUpdateCount();
}
+ // Flush all changes in batch if auto flush is true
+ flushIfNecessary();
// If we make it all the way through, clear the batch
clearBatch();
return returnCodes;
@@ -1321,9 +1323,17 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho
throw new SQLExceptionInfo.Builder(SQLExceptionCode.EXECUTE_UPDATE_WITH_NON_EMPTY_BATCH)
.build().buildException();
}
- return executeMutation(stmt);
+ int updateCount = executeMutation(stmt);
+ flushIfNecessary();
+ return updateCount;
}
+ private void flushIfNecessary() throws SQLException {
+ if (connection.getAutoFlush()) {
+ connection.flush();
+ }
+ }
+
@Override
public boolean execute(String sql) throws SQLException {
CompilableStatement stmt = parseStatement(sql);
@@ -1333,6 +1343,7 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho
.build().buildException();
}
executeMutation(stmt);
+ flushIfNecessary();
return false;
}
executeQuery(stmt);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cd4f4547/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 908c479..8f5f9a1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -86,7 +86,7 @@ public interface QueryServices extends SQLCloseable {
public static final String MUTATE_BATCH_SIZE_ATTRIB = "phoenix.mutate.batchSize";
public static final String MAX_SERVER_CACHE_TIME_TO_LIVE_MS_ATTRIB = "phoenix.coprocessor.maxServerCacheTimeToLiveMs";
- // Deprecated. Use FORCE_ROW_KEY_ORDER instead.
+ @Deprecated // Use FORCE_ROW_KEY_ORDER instead.
public static final String ROW_KEY_ORDER_SALTED_TABLE_ATTRIB = "phoenix.query.rowKeyOrderSaltedTable";
public static final String USE_INDEXES_ATTRIB = "phoenix.query.useIndexes";
@@ -164,9 +164,14 @@ public interface QueryServices extends SQLCloseable {
public static final String DEFAULT_KEEP_DELETED_CELLS_ATTRIB = "phoenix.table.default.keep.deleted.cells";
public static final String DEFAULT_STORE_NULLS_ATTRIB = "phoenix.table.default.store.nulls";
public static final String DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB = "phoenix.table.istransactional.default";
- public static final String TRANSACTIONS_ENABLED = "phoenix.transactions.enabled";
public static final String GLOBAL_METRICS_ENABLED = "phoenix.query.global.metrics.enabled";
+ // Transaction related configs
+ public static final String TRANSACTIONS_ENABLED = "phoenix.transactions.enabled";
+ // Controls whether or not uncommitted data is automatically sent to HBase
+ // at the end of a statement execution when transaction state is passed through.
+ public static final String AUTO_FLUSH_ATTRIB = "phoenix.transactions.autoFlush";
+
// rpc queue configs
public static final String INDEX_HANDLER_COUNT_ATTRIB = "phoenix.rpc.index.handler.count";
public static final String METADATA_HANDLER_COUNT_ATTRIB = "phoenix.rpc.metadata.handler.count";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cd4f4547/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 730a5ef..b256ff3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -71,8 +71,8 @@ import static org.apache.phoenix.query.QueryServices.USE_BYTE_BASED_REGEX_ATTRIB
import static org.apache.phoenix.query.QueryServices.USE_INDEXES_ATTRIB;
import java.util.HashSet;
-import java.util.Set;
import java.util.Map.Entry;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Coprocessor;
@@ -199,11 +199,13 @@ public class QueryServicesOptions {
// TODO Change this to true as part of PHOENIX-1543
// We'll also need this for transactions to work correctly
public static final boolean DEFAULT_AUTO_COMMIT = false;
- public static final boolean DEFAULT_TRANSACTIONAL = false;
public static final boolean DEFAULT_TABLE_ISTRANSACTIONAL = false;
public static final boolean DEFAULT_TRANSACTIONS_ENABLED = false;
public static final boolean DEFAULT_IS_GLOBAL_METRICS_ENABLED = true;
+ public static final boolean DEFAULT_TRANSACTIONAL = false;
+ public static final boolean DEFAULT_AUTO_FLUSH = false;
+
private static final String DEFAULT_CLIENT_RPC_CONTROLLER_FACTORY = ClientRpcControllerFactory.class.getName();
public static final String DEFAULT_CONSISTENCY_LEVEL = Consistency.STRONG.toString();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cd4f4547/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
index d4a45f6..a5fd96b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
@@ -98,14 +98,6 @@ import com.google.common.collect.Lists;
*/
public class PhoenixRuntime {
/**
- * Use this connection property to control HBase timestamps
- * by specifying your own long timestamp value at connection time. All
- * queries will use this as the upper bound of the time range for scans
- * and DDL, and DML will use this as t he timestamp for key values.
- */
- public static final String CURRENT_SCN_ATTRIB = "CurrentSCN";
-
- /**
* Root for the JDBC URL that the Phoenix accepts accepts.
*/
public final static String JDBC_PROTOCOL = "jdbc:phoenix";
@@ -121,13 +113,12 @@ public class PhoenixRuntime {
public final static String EMBEDDED_JDBC_PROTOCOL = PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
/**
- * Use this connection property to control the number of rows that are
- * batched together on an UPSERT INTO table1... SELECT ... FROM table2.
- * It's only used when autoCommit is true and your source table is
- * different than your target table or your SELECT statement has a
- * GROUP BY clause.
+ * Use this connection property to control HBase timestamps
+ * by specifying your own long timestamp value at connection time. All
+ * queries will use this as the upper bound of the time range for scans
+ * and DDL, and DML will use this as t he timestamp for key values.
*/
- public final static String UPSERT_BATCH_SIZE_ATTRIB = "UpsertBatchSize";
+ public static final String CURRENT_SCN_ATTRIB = "CurrentSCN";
/**
* Use this connection property to help with fairness of resource allocation
@@ -139,11 +130,13 @@ public class PhoenixRuntime {
public static final String TENANT_ID_ATTRIB = "TenantId";
/**
- * Use this connection property prefix for annotations that you want to show up in traces and log lines emitted by Phoenix.
- * This is useful for annotating connections with information available on the client (e.g. user or session identifier) and
- * having these annotation automatically passed into log lines and traces by Phoenix.
+ * Use this connection property to control the number of rows that are
+ * batched together on an UPSERT INTO table1... SELECT ... FROM table2.
+ * It's only used when autoCommit is true and your source table is
+ * different than your target table or your SELECT statement has a
+ * GROUP BY clause.
*/
- public static final String ANNOTATION_ATTRIB_PREFIX = "phoenix.annotation.";
+ public final static String UPSERT_BATCH_SIZE_ATTRIB = "UpsertBatchSize";
/**
* Use this connection property to explicitly enable or disable auto-commit on a new connection.
@@ -151,11 +144,38 @@ public class PhoenixRuntime {
public static final String AUTO_COMMIT_ATTRIB = "AutoCommit";
/**
+<<<<<<< HEAD
+=======
+<<<<<<< HEAD
+=======
+>>>>>>> 4e96747... PHOENIX-2411 Allow Phoenix to participate as transactional component
* Use this connection property to explicitly set read consistency level on a new connection.
*/
public static final String CONSISTENCY_ATTRIB = "Consistency";
/**
+<<<<<<< HEAD
+=======
+ * Use this connection property to explicitly enable or disable request level metric collection.
+ */
+ public static final String REQUEST_METRIC_ATTRIB = "RequestMetric";
+
+ /**
+ * All Phoenix specific connection properties
+ * TODO: use enum instead
+ */
+ public final static String[] CONNECTION_PROPERTIES = {
+ CURRENT_SCN_ATTRIB,
+ TENANT_ID_ATTRIB,
+ UPSERT_BATCH_SIZE_ATTRIB,
+ AUTO_COMMIT_ATTRIB,
+ CONSISTENCY_ATTRIB,
+ REQUEST_METRIC_ATTRIB
+ };
+
+ /**
+>>>>>>> 5180ae0... PHOENIX-2411 Allow Phoenix to participate as transactional component
+>>>>>>> 4e96747... PHOENIX-2411 Allow Phoenix to participate as transactional component
* Use this as the zookeeper quorum name to have a connection-less connection. This enables
* Phoenix-compatible HFiles to be created in a map/reduce job by creating tables,
* upserting data into them, and getting the uncommitted state through {@link #getUncommittedData(Connection)}
@@ -163,9 +183,11 @@ public class PhoenixRuntime {
public final static String CONNECTIONLESS = "none";
/**
- * Use this connection property to explicitly enable or disable request level metric collection.
+ * Use this connection property prefix for annotations that you want to show up in traces and log lines emitted by Phoenix.
+ * This is useful for annotating connections with information available on the client (e.g. user or session identifier) and
+ * having these annotation automatically passed into log lines and traces by Phoenix.
*/
- public static final String REQUEST_METRIC_ATTRIB = "RequestMetric";
+ public static final String ANNOTATION_ATTRIB_PREFIX = "phoenix.annotation.";
private static final String HEADER_IN_LINE = "in-line";
private static final String SQL_FILE_EXT = ".sql";