You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tephra.apache.org by go...@apache.org on 2017/03/07 00:34:14 UTC
[1/9] incubator-tephra git commit: [maven-release-plugin] prepare
release v0.11.0-incubating
Repository: incubator-tephra
Updated Branches:
refs/heads/master 45cc0e9f4 -> e02060620
[maven-release-plugin] prepare release v0.11.0-incubating
Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/d6327007
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/d6327007
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/d6327007
Branch: refs/heads/master
Commit: d6327007b6240c9b2605c2e8d91ca9ac92ecedfc
Parents: 203825e
Author: Gokul Gunasekaran <go...@cask.co>
Authored: Tue Feb 14 00:03:18 2017 -0800
Committer: Gokul Gunasekaran <go...@cask.co>
Committed: Tue Feb 14 00:03:18 2017 -0800
----------------------------------------------------------------------
pom.xml | 4 ++--
tephra-api/pom.xml | 2 +-
tephra-core/pom.xml | 2 +-
tephra-distribution/pom.xml | 2 +-
tephra-examples/cdh-5.7/pom.xml | 2 +-
tephra-examples/cdh-5.8/pom.xml | 2 +-
tephra-examples/hbase-0.96/pom.xml | 2 +-
tephra-examples/hbase-0.98/pom.xml | 2 +-
tephra-examples/hbase-1.0-cdh/pom.xml | 2 +-
tephra-examples/hbase-1.0/pom.xml | 2 +-
tephra-examples/hbase-1.1/pom.xml | 2 +-
tephra-examples/hbase-1.2/pom.xml | 2 +-
tephra-examples/pom.xml | 2 +-
tephra-hbase-compat-0.96/pom.xml | 2 +-
tephra-hbase-compat-0.98/pom.xml | 2 +-
tephra-hbase-compat-1.0-cdh/pom.xml | 2 +-
tephra-hbase-compat-1.0/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml | 2 +-
21 files changed, 22 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 90f9843..1649d74 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,7 +28,7 @@
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
<packaging>pom</packaging>
<name>Apache Tephra</name>
<description>
@@ -165,7 +165,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-tephra.git</connection>
<url>https://git-wip-us.apache.org/repos/asf?p=incubator-tephra.git;a=summary</url>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-tephra.git</developerConnection>
- <tag>HEAD</tag>
+ <tag>v0.11.0-incubating</tag>
</scm>
<issueManagement>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-api/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-api/pom.xml b/tephra-api/pom.xml
index cf3e020..0ed4e2e 100644
--- a/tephra-api/pom.xml
+++ b/tephra-api/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<artifactId>tephra-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-core/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-core/pom.xml b/tephra-core/pom.xml
index 1e4a0c8..8f7104d 100644
--- a/tephra-core/pom.xml
+++ b/tephra-core/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-distribution/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-distribution/pom.xml b/tephra-distribution/pom.xml
index 56bb7f9..6df3d8f 100644
--- a/tephra-distribution/pom.xml
+++ b/tephra-distribution/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<artifactId>tephra-distribution</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-examples/cdh-5.7/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/cdh-5.7/pom.xml b/tephra-examples/cdh-5.7/pom.xml
index aa2a33a..f83aad7 100644
--- a/tephra-examples/cdh-5.7/pom.xml
+++ b/tephra-examples/cdh-5.7/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-examples</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-examples/cdh-5.8/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/cdh-5.8/pom.xml b/tephra-examples/cdh-5.8/pom.xml
index ad1593d..bb10753 100644
--- a/tephra-examples/cdh-5.8/pom.xml
+++ b/tephra-examples/cdh-5.8/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-examples/hbase-0.96/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-0.96/pom.xml b/tephra-examples/hbase-0.96/pom.xml
index 1d1a656..e481c8f 100644
--- a/tephra-examples/hbase-0.96/pom.xml
+++ b/tephra-examples/hbase-0.96/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-examples/hbase-0.98/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-0.98/pom.xml b/tephra-examples/hbase-0.98/pom.xml
index 80f3a5c..4e59a8e 100644
--- a/tephra-examples/hbase-0.98/pom.xml
+++ b/tephra-examples/hbase-0.98/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-examples/hbase-1.0-cdh/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.0-cdh/pom.xml b/tephra-examples/hbase-1.0-cdh/pom.xml
index 5959fae..e8c7490 100644
--- a/tephra-examples/hbase-1.0-cdh/pom.xml
+++ b/tephra-examples/hbase-1.0-cdh/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-examples/hbase-1.0/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.0/pom.xml b/tephra-examples/hbase-1.0/pom.xml
index bd1989c..89597f2 100644
--- a/tephra-examples/hbase-1.0/pom.xml
+++ b/tephra-examples/hbase-1.0/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-examples/hbase-1.1/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.1/pom.xml b/tephra-examples/hbase-1.1/pom.xml
index 594cc6c..b377546 100644
--- a/tephra-examples/hbase-1.1/pom.xml
+++ b/tephra-examples/hbase-1.1/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-examples</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-examples/hbase-1.2/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.2/pom.xml b/tephra-examples/hbase-1.2/pom.xml
index 933210c..4284a91 100644
--- a/tephra-examples/hbase-1.2/pom.xml
+++ b/tephra-examples/hbase-1.2/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-examples</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-examples/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/pom.xml b/tephra-examples/pom.xml
index ad4d01a..3d6ad4e 100644
--- a/tephra-examples/pom.xml
+++ b/tephra-examples/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-hbase-compat-0.96/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/pom.xml b/tephra-hbase-compat-0.96/pom.xml
index dbadc2d..282f385 100644
--- a/tephra-hbase-compat-0.96/pom.xml
+++ b/tephra-hbase-compat-0.96/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-hbase-compat-0.98/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/pom.xml b/tephra-hbase-compat-0.98/pom.xml
index 21c3bc8..d278641 100644
--- a/tephra-hbase-compat-0.98/pom.xml
+++ b/tephra-hbase-compat-0.98/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-hbase-compat-1.0-cdh/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/pom.xml b/tephra-hbase-compat-1.0-cdh/pom.xml
index 738a2cc..2b5d70d 100644
--- a/tephra-hbase-compat-1.0-cdh/pom.xml
+++ b/tephra-hbase-compat-1.0-cdh/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-hbase-compat-1.0/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/pom.xml b/tephra-hbase-compat-1.0/pom.xml
index c7ec8ff..c81b708 100644
--- a/tephra-hbase-compat-1.0/pom.xml
+++ b/tephra-hbase-compat-1.0/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-hbase-compat-1.1-base/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/pom.xml b/tephra-hbase-compat-1.1-base/pom.xml
index 9eef513..71a414d 100644
--- a/tephra-hbase-compat-1.1-base/pom.xml
+++ b/tephra-hbase-compat-1.1-base/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
index 8b59d31..e6637c5 100644
--- a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
+++ b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.1-base</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
index 3143613..f36762c 100644
--- a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
+++ b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.1-base</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d6327007/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
index 854a745..4f5981f 100644
--- a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
+++ b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.1-base</artifactId>
- <version>0.11.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
[9/9] incubator-tephra git commit: Merge branch
'release/0.11.0-incubating'
Posted by go...@apache.org.
Merge branch 'release/0.11.0-incubating'
Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/e0206062
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/e0206062
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/e0206062
Branch: refs/heads/master
Commit: e0206062029c03b3e317ef885d25add95ab661b1
Parents: 45cc0e9 14aca68
Author: Gokul Gunasekaran <go...@cask.co>
Authored: Mon Mar 6 16:34:02 2017 -0800
Committer: Gokul Gunasekaran <go...@cask.co>
Committed: Mon Mar 6 16:34:02 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/tephra/TxConstants.java | 6 ++
.../txprune/TransactionPruningRunnable.java | 12 ++-
.../txprune/TransactionPruningService.java | 9 +-
.../txprune/TransactionPruningServiceTest.java | 10 +-
.../hbase/txprune/InvalidListPruningDebug.java | 105 +++++++++++++++++--
.../hbase/txprune/PruneUpperBoundWriter.java | 2 +-
.../hbase/txprune/InvalidListPruningDebug.java | 105 +++++++++++++++++--
.../hbase/txprune/PruneUpperBoundWriter.java | 18 +++-
.../hbase/txprune/InvalidListPruningDebug.java | 105 +++++++++++++++++--
.../hbase/txprune/PruneUpperBoundWriter.java | 18 +++-
.../hbase/txprune/InvalidListPruningDebug.java | 105 +++++++++++++++++--
.../hbase/txprune/PruneUpperBoundWriter.java | 18 +++-
.../hbase/txprune/InvalidListPruningDebug.java | 105 +++++++++++++++++--
13 files changed, 563 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
[7/9] incubator-tephra git commit: [maven-release-plugin] prepare
release v0.11.0-incubating
Posted by go...@apache.org.
[maven-release-plugin] prepare release v0.11.0-incubating
Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/a12cff8a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/a12cff8a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/a12cff8a
Branch: refs/heads/master
Commit: a12cff8a6d6cceaa0d05470bfe407bd2bccde327
Parents: 66a2fce
Author: Gokul Gunasekaran <go...@cask.co>
Authored: Mon Mar 6 15:33:54 2017 -0800
Committer: Gokul Gunasekaran <go...@cask.co>
Committed: Mon Mar 6 15:33:54 2017 -0800
----------------------------------------------------------------------
pom.xml | 4 ++--
tephra-api/pom.xml | 2 +-
tephra-core/pom.xml | 2 +-
tephra-distribution/pom.xml | 2 +-
tephra-examples/cdh-5.7/pom.xml | 2 +-
tephra-examples/cdh-5.8/pom.xml | 2 +-
tephra-examples/hbase-0.96/pom.xml | 2 +-
tephra-examples/hbase-0.98/pom.xml | 2 +-
tephra-examples/hbase-1.0-cdh/pom.xml | 2 +-
tephra-examples/hbase-1.0/pom.xml | 2 +-
tephra-examples/hbase-1.1/pom.xml | 2 +-
tephra-examples/hbase-1.2/pom.xml | 2 +-
tephra-examples/pom.xml | 2 +-
tephra-hbase-compat-0.96/pom.xml | 2 +-
tephra-hbase-compat-0.98/pom.xml | 2 +-
tephra-hbase-compat-1.0-cdh/pom.xml | 2 +-
tephra-hbase-compat-1.0/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml | 2 +-
21 files changed, 22 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e81dad4..1649d74 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,7 +28,7 @@
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
<packaging>pom</packaging>
<name>Apache Tephra</name>
<description>
@@ -165,7 +165,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-tephra.git</connection>
<url>https://git-wip-us.apache.org/repos/asf?p=incubator-tephra.git;a=summary</url>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-tephra.git</developerConnection>
- <tag>HEAD</tag>
+ <tag>v0.11.0-incubating</tag>
</scm>
<issueManagement>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-api/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-api/pom.xml b/tephra-api/pom.xml
index 9f386b2..0ed4e2e 100644
--- a/tephra-api/pom.xml
+++ b/tephra-api/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<artifactId>tephra-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-core/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-core/pom.xml b/tephra-core/pom.xml
index 54c5440..8f7104d 100644
--- a/tephra-core/pom.xml
+++ b/tephra-core/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-distribution/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-distribution/pom.xml b/tephra-distribution/pom.xml
index e440ece..6df3d8f 100644
--- a/tephra-distribution/pom.xml
+++ b/tephra-distribution/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<artifactId>tephra-distribution</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-examples/cdh-5.7/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/cdh-5.7/pom.xml b/tephra-examples/cdh-5.7/pom.xml
index 53efe12..f83aad7 100644
--- a/tephra-examples/cdh-5.7/pom.xml
+++ b/tephra-examples/cdh-5.7/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-examples</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-examples/cdh-5.8/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/cdh-5.8/pom.xml b/tephra-examples/cdh-5.8/pom.xml
index f29476e..bb10753 100644
--- a/tephra-examples/cdh-5.8/pom.xml
+++ b/tephra-examples/cdh-5.8/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-examples/hbase-0.96/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-0.96/pom.xml b/tephra-examples/hbase-0.96/pom.xml
index 8c8d87d..e481c8f 100644
--- a/tephra-examples/hbase-0.96/pom.xml
+++ b/tephra-examples/hbase-0.96/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-examples/hbase-0.98/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-0.98/pom.xml b/tephra-examples/hbase-0.98/pom.xml
index f3de4f5..4e59a8e 100644
--- a/tephra-examples/hbase-0.98/pom.xml
+++ b/tephra-examples/hbase-0.98/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-examples/hbase-1.0-cdh/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.0-cdh/pom.xml b/tephra-examples/hbase-1.0-cdh/pom.xml
index 7382b87..e8c7490 100644
--- a/tephra-examples/hbase-1.0-cdh/pom.xml
+++ b/tephra-examples/hbase-1.0-cdh/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-examples/hbase-1.0/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.0/pom.xml b/tephra-examples/hbase-1.0/pom.xml
index 52a8717..89597f2 100644
--- a/tephra-examples/hbase-1.0/pom.xml
+++ b/tephra-examples/hbase-1.0/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-examples/hbase-1.1/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.1/pom.xml b/tephra-examples/hbase-1.1/pom.xml
index e15574c..b377546 100644
--- a/tephra-examples/hbase-1.1/pom.xml
+++ b/tephra-examples/hbase-1.1/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-examples</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-examples/hbase-1.2/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.2/pom.xml b/tephra-examples/hbase-1.2/pom.xml
index 2cef197..4284a91 100644
--- a/tephra-examples/hbase-1.2/pom.xml
+++ b/tephra-examples/hbase-1.2/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-examples</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-examples/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/pom.xml b/tephra-examples/pom.xml
index 63d0a66..3d6ad4e 100644
--- a/tephra-examples/pom.xml
+++ b/tephra-examples/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-hbase-compat-0.96/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/pom.xml b/tephra-hbase-compat-0.96/pom.xml
index 0efc506..282f385 100644
--- a/tephra-hbase-compat-0.96/pom.xml
+++ b/tephra-hbase-compat-0.96/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-hbase-compat-0.98/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/pom.xml b/tephra-hbase-compat-0.98/pom.xml
index b83372b..d278641 100644
--- a/tephra-hbase-compat-0.98/pom.xml
+++ b/tephra-hbase-compat-0.98/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-hbase-compat-1.0-cdh/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/pom.xml b/tephra-hbase-compat-1.0-cdh/pom.xml
index 31dbf56..2b5d70d 100644
--- a/tephra-hbase-compat-1.0-cdh/pom.xml
+++ b/tephra-hbase-compat-1.0-cdh/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-hbase-compat-1.0/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/pom.xml b/tephra-hbase-compat-1.0/pom.xml
index 84e203d..c81b708 100644
--- a/tephra-hbase-compat-1.0/pom.xml
+++ b/tephra-hbase-compat-1.0/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-hbase-compat-1.1-base/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/pom.xml b/tephra-hbase-compat-1.1-base/pom.xml
index 1ae7829..71a414d 100644
--- a/tephra-hbase-compat-1.1-base/pom.xml
+++ b/tephra-hbase-compat-1.1-base/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
index 0ac15be..e6637c5 100644
--- a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
+++ b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.1-base</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
index b152305..f36762c 100644
--- a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
+++ b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.1-base</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a12cff8a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
index 359af98..4f5981f 100644
--- a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
+++ b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.1-base</artifactId>
- <version>0.12.0-incubating-SNAPSHOT</version>
+ <version>0.11.0-incubating</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
[3/9] incubator-tephra git commit: TEPHRA-223 Encapsulate the two
data structures used for invalid transactions to avoid update issues
Posted by go...@apache.org.
TEPHRA-223 Encapsulate the two data structures used for invalid transactions to avoid update issues
This closes #37
Signed-off-by: poorna <po...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/872fb109
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/872fb109
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/872fb109
Branch: refs/heads/master
Commit: 872fb1090efadb3d62a75fa8f51b308511eea754
Parents: 10e36e6
Author: poorna <po...@cask.co>
Authored: Mon Feb 20 17:13:39 2017 -0800
Committer: poorna <po...@apache.org>
Committed: Tue Feb 21 17:46:33 2017 -0800
----------------------------------------------------------------------
.../org/apache/tephra/TransactionManager.java | 104 +++++++--------
.../apache/tephra/manager/InvalidTxList.java | 126 +++++++++++++++++++
.../tephra/manager/InvalidTxListTest.java | 110 ++++++++++++++++
.../AbstractTransactionStateStorageTest.java | 10 ++
4 files changed, 294 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/872fb109/tephra-core/src/main/java/org/apache/tephra/TransactionManager.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/TransactionManager.java b/tephra-core/src/main/java/org/apache/tephra/TransactionManager.java
index 0b90d7f..f2060cd 100644
--- a/tephra-core/src/main/java/org/apache/tephra/TransactionManager.java
+++ b/tephra-core/src/main/java/org/apache/tephra/TransactionManager.java
@@ -29,7 +29,11 @@ import com.google.common.collect.Sets;
import com.google.common.util.concurrent.AbstractService;
import com.google.inject.Inject;
import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.longs.LongArraySet;
+import it.unimi.dsi.fastutil.longs.LongIterator;
+import it.unimi.dsi.fastutil.longs.LongSet;
import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.manager.InvalidTxList;
import org.apache.tephra.metrics.DefaultMetricsCollector;
import org.apache.tephra.metrics.MetricsCollector;
import org.apache.tephra.persist.NoOpTransactionStateStorage;
@@ -46,7 +50,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
@@ -115,15 +118,11 @@ public class TransactionManager extends AbstractService {
//poll every 10 second to emit metrics
private static final long METRICS_POLL_INTERVAL = 10000L;
- private static final long[] NO_INVALID_TX = { };
-
// Transactions that are in progress, with their info.
private final NavigableMap<Long, InProgressTx> inProgress = new ConcurrentSkipListMap<Long, InProgressTx>();
// the list of transactions that are invalid (not properly committed/aborted, or timed out)
- // TODO: explain usage of two arrays
- private final LongArrayList invalid = new LongArrayList();
- private long[] invalidArray = NO_INVALID_TX;
+ private final InvalidTxList invalidTxList = new InvalidTxList();
// todo: use moving array instead (use Long2ObjectMap<byte[]> in fastutil)
// todo: should this be consolidated with inProgress?
@@ -198,8 +197,7 @@ public class TransactionManager extends AbstractService {
}
private void clear() {
- invalid.clear();
- invalidArray = NO_INVALID_TX;
+ invalidTxList.clear();
inProgress.clear();
committedChangeSets.clear();
committingChangeSets.clear();
@@ -318,7 +316,7 @@ public class TransactionManager extends AbstractService {
txMetricsCollector.gauge("committing.size", committingChangeSets.size());
txMetricsCollector.gauge("committed.size", committedChangeSets.size());
txMetricsCollector.gauge("inprogress.size", inProgress.size());
- txMetricsCollector.gauge("invalid.size", invalidArray.length);
+ txMetricsCollector.gauge("invalid.size", getInvalidSize());
}
@Override
@@ -327,7 +325,7 @@ public class TransactionManager extends AbstractService {
txMetricsCollector.gauge("committing.size", committingChangeSets.size());
txMetricsCollector.gauge("committed.size", committedChangeSets.size());
txMetricsCollector.gauge("inprogress.size", inProgress.size());
- txMetricsCollector.gauge("invalid.size", invalidArray.length);
+ txMetricsCollector.gauge("invalid.size", getInvalidSize());
}
@Override
@@ -363,7 +361,7 @@ public class TransactionManager extends AbstractService {
}
if (!timedOut.isEmpty()) {
invalidEdits = Lists.newArrayListWithCapacity(timedOut.size());
- invalid.addAll(timedOut.keySet());
+ invalidTxList.addAll(timedOut.keySet());
for (Map.Entry<Long, InProgressType> tx : timedOut.entrySet()) {
inProgress.remove(tx.getKey());
// checkpoints never go into the committing change sets or the edits
@@ -373,9 +371,6 @@ public class TransactionManager extends AbstractService {
}
}
- // todo: find a more efficient way to keep this sorted. Could it just be an array?
- Collections.sort(invalid);
- invalidArray = invalid.toLongArray();
LOG.info("Invalidated {} transactions due to timeout.", timedOut.size());
}
}
@@ -468,7 +463,8 @@ public class TransactionManager extends AbstractService {
public synchronized TransactionSnapshot getCurrentState() {
return TransactionSnapshot.copyFrom(System.currentTimeMillis(), readPointer, lastWritePointer,
- invalid, inProgress, committingChangeSets, committedChangeSets);
+ invalidTxList.toRawList(), inProgress, committingChangeSets,
+ committedChangeSets);
}
public synchronized void recoverState() {
@@ -497,7 +493,7 @@ public class TransactionManager extends AbstractService {
Preconditions.checkState(lastSnapshotTime == 0, "lastSnapshotTime has been set!");
Preconditions.checkState(readPointer == 0, "readPointer has been set!");
Preconditions.checkState(lastWritePointer == 0, "lastWritePointer has been set!");
- Preconditions.checkState(invalid.isEmpty(), "invalid list should be empty!");
+ Preconditions.checkState(invalidTxList.isEmpty(), "invalid list should be empty!");
Preconditions.checkState(inProgress.isEmpty(), "inProgress map should be empty!");
Preconditions.checkState(committingChangeSets.isEmpty(), "committingChangeSets should be empty!");
Preconditions.checkState(committedChangeSets.isEmpty(), "committedChangeSets should be empty!");
@@ -506,7 +502,7 @@ public class TransactionManager extends AbstractService {
lastSnapshotTime = snapshot.getTimestamp();
readPointer = snapshot.getReadPointer();
lastWritePointer = snapshot.getWritePointer();
- invalid.addAll(snapshot.getInvalid());
+ invalidTxList.addAll(snapshot.getInvalid());
inProgress.putAll(txnBackwardsCompatCheck(defaultLongTimeout, longTimeoutTolerance, snapshot.getInProgress()));
committingChangeSets.putAll(snapshot.getCommittingChangeSets());
committedChangeSets.putAll(snapshot.getCommittedChangeSets());
@@ -818,14 +814,17 @@ public class TransactionManager extends AbstractService {
txMetricsCollector.rate("canCommit");
Stopwatch timer = new Stopwatch().start();
if (inProgress.get(tx.getTransactionId()) == null) {
- // invalid transaction, either this has timed out and moved to invalid, or something else is wrong.
- if (invalid.contains(tx.getTransactionId())) {
- throw new TransactionNotInProgressException(
- String.format("canCommit() is called for transaction %d that is not in progress (it is known to be invalid)",
- tx.getTransactionId()));
- } else {
- throw new TransactionNotInProgressException(
- String.format("canCommit() is called for transaction %d that is not in progress", tx.getTransactionId()));
+ synchronized (this) {
+ // invalid transaction, either this has timed out and moved to invalid, or something else is wrong.
+ if (invalidTxList.contains(tx.getTransactionId())) {
+ throw new TransactionNotInProgressException(
+ String.format(
+ "canCommit() is called for transaction %d that is not in progress (it is known to be invalid)",
+ tx.getTransactionId()));
+ } else {
+ throw new TransactionNotInProgressException(
+ String.format("canCommit() is called for transaction %d that is not in progress", tx.getTransactionId()));
+ }
}
}
@@ -872,7 +871,7 @@ public class TransactionManager extends AbstractService {
commitPointer = lastWritePointer + 1;
if (inProgress.get(tx.getTransactionId()) == null) {
// invalid transaction, either this has timed out and moved to invalid, or something else is wrong.
- if (invalid.contains(tx.getTransactionId())) {
+ if (invalidTxList.contains(tx.getTransactionId())) {
throw new TransactionNotInProgressException(
String.format("canCommit() is called for transaction %d that is not in progress " +
"(it is known to be invalid)", tx.getTransactionId()));
@@ -928,8 +927,7 @@ public class TransactionManager extends AbstractService {
InProgressTx previous = inProgress.remove(transactionId);
if (previous == null) {
// tx was not in progress! perhaps it timed out and is invalid? try to remove it there.
- if (invalid.rem(transactionId)) {
- invalidArray = invalid.toLongArray();
+ if (invalidTxList.remove(transactionId)) {
LOG.info("Tx invalid list: removed committed tx {}", transactionId);
}
} else {
@@ -983,17 +981,16 @@ public class TransactionManager extends AbstractService {
boolean removeInProgressCheckpoints = true;
if (removed == null) {
// tx was not in progress! perhaps it timed out and is invalid? try to remove it there.
- if (invalid.rem(writePointer)) {
+ if (invalidTxList.remove(writePointer)) {
// the tx and all its children were invalidated: no need to remove them from inProgress
removeInProgressCheckpoints = false;
// remove any invalidated checkpoint pointers
// this will only be present if the parent write pointer was also invalidated
if (checkpointWritePointers != null) {
for (long checkpointWritePointer : checkpointWritePointers) {
- invalid.rem(checkpointWritePointer);
+ invalidTxList.remove(checkpointWritePointer);
}
}
- invalidArray = invalid.toLongArray();
LOG.info("Tx invalid list: removed aborted tx {}", writePointer);
}
}
@@ -1032,21 +1029,18 @@ public class TransactionManager extends AbstractService {
// This check is to prevent from invalidating committed transactions
if (previous != null || previousChangeSet != null) {
// add tx to invalids
- invalid.add(writePointer);
+ invalidTxList.add(writePointer);
if (previous == null) {
LOG.debug("Invalidating tx {} in committing change sets but not in-progress", writePointer);
} else {
// invalidate any checkpoint write pointers
LongArrayList childWritePointers = previous.getCheckpointWritePointers();
if (!childWritePointers.isEmpty()) {
- invalid.addAll(childWritePointers);
+ invalidTxList.addAll(childWritePointers);
inProgress.keySet().removeAll(childWritePointers);
}
}
LOG.info("Tx invalid list: added tx {} because of invalidate", writePointer);
- // todo: find a more efficient way to keep this sorted. Could it just be an array?
- Collections.sort(invalid);
- invalidArray = invalid.toLongArray();
if (previous != null && !previous.isLongRunning()) {
// tx was short-running: must move read pointer
moveReadPointerIfNeeded(writePointer);
@@ -1080,13 +1074,9 @@ public class TransactionManager extends AbstractService {
}
}
- private boolean doTruncateInvalidTx(Set<Long> invalidTxIds) {
- LOG.info("Removing tx ids {} from invalid list", invalidTxIds);
- boolean success = invalid.removeAll(invalidTxIds);
- if (success) {
- invalidArray = invalid.toLongArray();
- }
- return success;
+ private boolean doTruncateInvalidTx(Set<Long> toRemove) {
+ LOG.info("Removing tx ids {} from invalid list", toRemove);
+ return invalidTxList.removeAll(toRemove);
}
/**
@@ -1123,15 +1113,16 @@ public class TransactionManager extends AbstractService {
}
// Find all invalid transactions earlier than truncateWp
- Set<Long> toTruncate = Sets.newHashSet();
- for (long wp : invalid) {
- // invalid list is sorted, hence can stop as soon as we reach a wp >= truncateWp
- if (wp >= truncateWp) {
- break;
+ LongSet toTruncate = new LongArraySet();
+ LongIterator it = invalidTxList.toRawList().iterator();
+ while (it.hasNext()) {
+ long wp = it.nextLong();
+ if (wp < truncateWp) {
+ toTruncate.add(wp);
}
- toTruncate.add(wp);
}
- return doTruncateInvalidTx(toTruncate);
+ LOG.info("Removing tx ids {} from invalid list", toTruncate);
+ return invalidTxList.removeAll(toTruncate);
}
public Transaction checkpoint(Transaction originalTx) throws TransactionNotInProgressException {
@@ -1149,7 +1140,7 @@ public class TransactionManager extends AbstractService {
// check that the parent tx is in progress
InProgressTx parentTx = inProgress.get(txId);
if (parentTx == null) {
- if (invalid.contains(txId)) {
+ if (invalidTxList.contains(txId)) {
throw new TransactionNotInProgressException(
String.format("Transaction %d is not in progress because it was invalidated", txId));
} else {
@@ -1184,14 +1175,14 @@ public class TransactionManager extends AbstractService {
// hack for exposing important metric
public int getExcludedListSize() {
- return invalid.size() + inProgress.size();
+ return getInvalidSize() + inProgress.size();
}
/**
* @return the size of invalid list
*/
- public int getInvalidSize() {
- return this.invalid.size();
+ public synchronized int getInvalidSize() {
+ return this.invalidTxList.size();
}
int getCommittedSize() {
@@ -1254,7 +1245,8 @@ public class TransactionManager extends AbstractService {
firstShortTx = txId;
}
}
- return new Transaction(readPointer, writePointer, invalidArray, inProgressIds.toLongArray(), firstShortTx, type);
+ return new Transaction(readPointer, writePointer, invalidTxList.toSortedArray(),
+ inProgressIds.toLongArray(), firstShortTx, type);
}
private void appendToLog(TransactionEdit edit) {
@@ -1285,7 +1277,7 @@ public class TransactionManager extends AbstractService {
*/
public void logStatistics() {
LOG.info("Transaction Statistics: write pointer = " + lastWritePointer +
- ", invalid = " + invalid.size() +
+ ", invalid = " + getInvalidSize() +
", in progress = " + inProgress.size() +
", committing = " + committingChangeSets.size() +
", committed = " + committedChangeSets.size());
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/872fb109/tephra-core/src/main/java/org/apache/tephra/manager/InvalidTxList.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/manager/InvalidTxList.java b/tephra-core/src/main/java/org/apache/tephra/manager/InvalidTxList.java
new file mode 100644
index 0000000..231196c
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/manager/InvalidTxList.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tephra.manager;
+
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.longs.LongList;
+import it.unimi.dsi.fastutil.longs.LongLists;
+import org.apache.tephra.TransactionManager;
+
+import java.util.Arrays;
+import java.util.Collection;
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * This is an internal class used by the {@link TransactionManager} to store invalid transaction ids.
+ * This class uses both a list and an array to keep track of the invalid ids. The list is the primary
+ * data structure for storing the invalid ids. The array is populated lazily on changes to the list.
+ * The array is used to avoid creating a new array every time method {@link #toSortedArray()} is invoked.
+ *
+ * This class is not thread safe and relies on external synchronization. TransactionManager always
+ * accesses an instance of this class after synchronization.
+ */
+@NotThreadSafe
+public class InvalidTxList {
+ private static final long[] NO_INVALID_TX = { };
+
+ private final LongList invalid = new LongArrayList();
+ private long[] invalidArray = NO_INVALID_TX;
+
+ private boolean dirty = false; // used to track changes to the invalid list
+
+ public int size() {
+ return invalid.size();
+ }
+
+ public boolean isEmpty() {
+ return invalid.isEmpty();
+ }
+
+ public boolean add(long id) {
+ boolean changed = invalid.add(id);
+ dirty = dirty || changed;
+ return changed;
+ }
+
+ public boolean addAll(Collection<? extends Long> ids) {
+ boolean changed = invalid.addAll(ids);
+ dirty = dirty || changed;
+ return changed;
+ }
+
+ public boolean addAll(LongList ids) {
+ boolean changed = invalid.addAll(ids);
+ dirty = dirty || changed;
+ return changed;
+ }
+
+ public boolean contains(long id) {
+ return invalid.contains(id);
+ }
+
+ public boolean remove(long id) {
+ boolean changed = invalid.rem(id);
+ dirty = dirty || changed;
+ return changed;
+ }
+
+ public boolean removeAll(Collection<? extends Long> ids) {
+ boolean changed = invalid.removeAll(ids);
+ dirty = dirty || changed;
+ return changed;
+ }
+
+ @SuppressWarnings("WeakerAccess")
+ public boolean removeAll(LongList ids) {
+ boolean changed = invalid.removeAll(ids);
+ dirty = dirty || changed;
+ return changed;
+ }
+
+ public void clear() {
+ invalid.clear();
+ invalidArray = NO_INVALID_TX;
+ dirty = false;
+ }
+
+ /**
+ * @return sorted array of invalid transactions
+ */
+ public long[] toSortedArray() {
+ lazyUpdate();
+ return invalidArray;
+ }
+
+ /**
+ * @return list of invalid transactions. The list is not sorted.
+ */
+ public LongList toRawList() {
+ return LongLists.unmodifiable(invalid);
+ }
+
+ private void lazyUpdate() {
+ if (dirty) {
+ invalidArray = invalid.toLongArray();
+ Arrays.sort(invalidArray);
+ dirty = false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/872fb109/tephra-core/src/test/java/org/apache/tephra/manager/InvalidTxListTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/manager/InvalidTxListTest.java b/tephra-core/src/test/java/org/apache/tephra/manager/InvalidTxListTest.java
new file mode 100644
index 0000000..4f45072
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/manager/InvalidTxListTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tephra.manager;
+
+import com.google.common.collect.ImmutableList;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class InvalidTxListTest {
+
+ @Test
+ public void testInvalidTxList() {
+ InvalidTxList invalidTxList = new InvalidTxList();
+ // Assert that the list is empty at the beginning
+ Assert.assertTrue(invalidTxList.isEmpty());
+ Assert.assertEquals(0, invalidTxList.size());
+ Assert.assertEquals(ImmutableList.of(), invalidTxList.toRawList());
+ Assert.assertArrayEquals(new long[0], invalidTxList.toSortedArray());
+
+ // Try removing something from the empty list
+ Assert.assertFalse(invalidTxList.remove(3));
+ Assert.assertFalse(invalidTxList.removeAll(ImmutableList.of(5L, 9L)));
+
+ // verify contains
+ Assert.assertFalse(invalidTxList.contains(3));
+
+ // Add some elements to the list
+ invalidTxList.add(3);
+ invalidTxList.add(1);
+ invalidTxList.add(8);
+ invalidTxList.add(5);
+
+ // verify contains
+ Assert.assertTrue(invalidTxList.contains(3));
+
+ // Assert the newly added elements
+ Assert.assertFalse(invalidTxList.isEmpty());
+ Assert.assertEquals(4, invalidTxList.size());
+ Assert.assertEquals(ImmutableList.of(3L, 1L, 8L, 5L), invalidTxList.toRawList());
+ Assert.assertArrayEquals(new long[] {1, 3, 5, 8}, invalidTxList.toSortedArray());
+
+ // Add a collection of elements
+ invalidTxList.addAll(ImmutableList.of(7L, 10L, 4L, 2L));
+
+ // Assert the newly added elements
+ Assert.assertFalse(invalidTxList.isEmpty());
+ Assert.assertEquals(8, invalidTxList.size());
+ Assert.assertEquals(ImmutableList.of(3L, 1L, 8L, 5L, 7L, 10L, 4L, 2L), invalidTxList.toRawList());
+ Assert.assertArrayEquals(new long[] {1, 2, 3, 4, 5, 7, 8, 10}, invalidTxList.toSortedArray());
+
+ // Remove elements that are not present
+ Assert.assertFalse(invalidTxList.remove(6));
+ Assert.assertFalse(invalidTxList.removeAll(ImmutableList.of(9L, 11L)));
+
+ // Remove a collection of elements
+ Assert.assertTrue(invalidTxList.removeAll(ImmutableList.of(8L, 4L, 2L)));
+ // This time check the array first and then check the list
+ Assert.assertArrayEquals(new long[] {1, 3, 5, 7, 10}, invalidTxList.toSortedArray());
+ Assert.assertEquals(ImmutableList.of(3L, 1L, 5L, 7L, 10L), invalidTxList.toRawList());
+
+ // Remove a single element
+ Assert.assertTrue(invalidTxList.remove(5));
+ Assert.assertArrayEquals(new long[] {1, 3, 7, 10}, invalidTxList.toSortedArray());
+ Assert.assertEquals(ImmutableList.of(3L, 1L, 7L, 10L), invalidTxList.toRawList());
+
+ // Add a LongCollection
+ invalidTxList.addAll(new LongArrayList(new long[] {15, 12, 13}));
+
+ // Assert the newly added elements
+ Assert.assertEquals(7, invalidTxList.size());
+ Assert.assertArrayEquals(new long[] {1, 3, 7, 10, 12, 13, 15}, invalidTxList.toSortedArray());
+ Assert.assertEquals(ImmutableList.of(3L, 1L, 7L, 10L, 15L, 12L, 13L), invalidTxList.toRawList());
+
+ // Remove a LongCollection
+ invalidTxList.removeAll(new LongArrayList(new long[] {3, 7, 12}));
+
+ // Assert removals
+ Assert.assertEquals(4, invalidTxList.size());
+ Assert.assertArrayEquals(new long[] {1, 10, 13, 15}, invalidTxList.toSortedArray());
+ Assert.assertEquals(ImmutableList.of(1L, 10L, 15L, 13L), invalidTxList.toRawList());
+
+ // Clear the list
+ invalidTxList.clear();
+ Assert.assertTrue(invalidTxList.isEmpty());
+ Assert.assertEquals(0, invalidTxList.size());
+ Assert.assertArrayEquals(new long[0], invalidTxList.toSortedArray());
+ Assert.assertEquals(ImmutableList.of(), invalidTxList.toRawList());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/872fb109/tephra-core/src/test/java/org/apache/tephra/persist/AbstractTransactionStateStorageTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/persist/AbstractTransactionStateStorageTest.java b/tephra-core/src/test/java/org/apache/tephra/persist/AbstractTransactionStateStorageTest.java
index 21090c5..ec06528 100644
--- a/tephra-core/src/test/java/org/apache/tephra/persist/AbstractTransactionStateStorageTest.java
+++ b/tephra-core/src/test/java/org/apache/tephra/persist/AbstractTransactionStateStorageTest.java
@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import com.google.common.primitives.Longs;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.apache.hadoop.conf.Configuration;
import org.apache.tephra.ChangeId;
@@ -137,6 +138,9 @@ public abstract class AbstractTransactionStateStorageTest {
// TODO: replace with new persistence tests
final byte[] a = { 'a' };
final byte[] b = { 'b' };
+ // Start and invalidate a transaction
+ Transaction invalid = txManager.startShort();
+ txManager.invalidate(invalid.getTransactionId());
// start a tx1, add a change A and commit
Transaction tx1 = txManager.startShort();
Assert.assertTrue(txManager.canCommit(tx1, Collections.singleton(a)));
@@ -162,6 +166,12 @@ public abstract class AbstractTransactionStateStorageTest {
LOG.info("New state: " + newState);
assertEquals(origState, newState);
+ // Verify that the invalid transaction list matches
+ Transaction checkTx = txManager.startShort();
+ Assert.assertEquals(origState.getInvalid(), Longs.asList(checkTx.getInvalids()));
+ txManager.abort(checkTx);
+ txManager.abort(invalid);
+
// commit tx2
Assert.assertTrue(txManager.commit(tx2));
// start another transaction, must be greater than tx3
[2/9] incubator-tephra git commit: [maven-release-plugin] prepare for
next development iteration
Posted by go...@apache.org.
[maven-release-plugin] prepare for next development iteration
Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/10e36e63
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/10e36e63
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/10e36e63
Branch: refs/heads/master
Commit: 10e36e632dbfc40eab5a2ed0d768975ac231adc1
Parents: d632700
Author: Gokul Gunasekaran <go...@cask.co>
Authored: Tue Feb 14 00:03:33 2017 -0800
Committer: Gokul Gunasekaran <go...@cask.co>
Committed: Tue Feb 14 00:03:33 2017 -0800
----------------------------------------------------------------------
pom.xml | 4 ++--
tephra-api/pom.xml | 2 +-
tephra-core/pom.xml | 2 +-
tephra-distribution/pom.xml | 2 +-
tephra-examples/cdh-5.7/pom.xml | 2 +-
tephra-examples/cdh-5.8/pom.xml | 2 +-
tephra-examples/hbase-0.96/pom.xml | 2 +-
tephra-examples/hbase-0.98/pom.xml | 2 +-
tephra-examples/hbase-1.0-cdh/pom.xml | 2 +-
tephra-examples/hbase-1.0/pom.xml | 2 +-
tephra-examples/hbase-1.1/pom.xml | 2 +-
tephra-examples/hbase-1.2/pom.xml | 2 +-
tephra-examples/pom.xml | 2 +-
tephra-hbase-compat-0.96/pom.xml | 2 +-
tephra-hbase-compat-0.98/pom.xml | 2 +-
tephra-hbase-compat-1.0-cdh/pom.xml | 2 +-
tephra-hbase-compat-1.0/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml | 2 +-
21 files changed, 22 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1649d74..e81dad4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,7 +28,7 @@
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Tephra</name>
<description>
@@ -165,7 +165,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-tephra.git</connection>
<url>https://git-wip-us.apache.org/repos/asf?p=incubator-tephra.git;a=summary</url>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-tephra.git</developerConnection>
- <tag>v0.11.0-incubating</tag>
+ <tag>HEAD</tag>
</scm>
<issueManagement>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-api/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-api/pom.xml b/tephra-api/pom.xml
index 0ed4e2e..9f386b2 100644
--- a/tephra-api/pom.xml
+++ b/tephra-api/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>tephra-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-core/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-core/pom.xml b/tephra-core/pom.xml
index 8f7104d..54c5440 100644
--- a/tephra-core/pom.xml
+++ b/tephra-core/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-distribution/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-distribution/pom.xml b/tephra-distribution/pom.xml
index 6df3d8f..e440ece 100644
--- a/tephra-distribution/pom.xml
+++ b/tephra-distribution/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>tephra-distribution</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-examples/cdh-5.7/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/cdh-5.7/pom.xml b/tephra-examples/cdh-5.7/pom.xml
index f83aad7..53efe12 100644
--- a/tephra-examples/cdh-5.7/pom.xml
+++ b/tephra-examples/cdh-5.7/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-examples</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-examples/cdh-5.8/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/cdh-5.8/pom.xml b/tephra-examples/cdh-5.8/pom.xml
index bb10753..f29476e 100644
--- a/tephra-examples/cdh-5.8/pom.xml
+++ b/tephra-examples/cdh-5.8/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-examples/hbase-0.96/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-0.96/pom.xml b/tephra-examples/hbase-0.96/pom.xml
index e481c8f..8c8d87d 100644
--- a/tephra-examples/hbase-0.96/pom.xml
+++ b/tephra-examples/hbase-0.96/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-examples/hbase-0.98/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-0.98/pom.xml b/tephra-examples/hbase-0.98/pom.xml
index 4e59a8e..f3de4f5 100644
--- a/tephra-examples/hbase-0.98/pom.xml
+++ b/tephra-examples/hbase-0.98/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-examples/hbase-1.0-cdh/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.0-cdh/pom.xml b/tephra-examples/hbase-1.0-cdh/pom.xml
index e8c7490..7382b87 100644
--- a/tephra-examples/hbase-1.0-cdh/pom.xml
+++ b/tephra-examples/hbase-1.0-cdh/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-examples/hbase-1.0/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.0/pom.xml b/tephra-examples/hbase-1.0/pom.xml
index 89597f2..52a8717 100644
--- a/tephra-examples/hbase-1.0/pom.xml
+++ b/tephra-examples/hbase-1.0/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-examples/hbase-1.1/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.1/pom.xml b/tephra-examples/hbase-1.1/pom.xml
index b377546..e15574c 100644
--- a/tephra-examples/hbase-1.1/pom.xml
+++ b/tephra-examples/hbase-1.1/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-examples</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-examples/hbase-1.2/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.2/pom.xml b/tephra-examples/hbase-1.2/pom.xml
index 4284a91..2cef197 100644
--- a/tephra-examples/hbase-1.2/pom.xml
+++ b/tephra-examples/hbase-1.2/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-examples</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-examples/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/pom.xml b/tephra-examples/pom.xml
index 3d6ad4e..63d0a66 100644
--- a/tephra-examples/pom.xml
+++ b/tephra-examples/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-hbase-compat-0.96/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/pom.xml b/tephra-hbase-compat-0.96/pom.xml
index 282f385..0efc506 100644
--- a/tephra-hbase-compat-0.96/pom.xml
+++ b/tephra-hbase-compat-0.96/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-hbase-compat-0.98/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/pom.xml b/tephra-hbase-compat-0.98/pom.xml
index d278641..b83372b 100644
--- a/tephra-hbase-compat-0.98/pom.xml
+++ b/tephra-hbase-compat-0.98/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-hbase-compat-1.0-cdh/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/pom.xml b/tephra-hbase-compat-1.0-cdh/pom.xml
index 2b5d70d..31dbf56 100644
--- a/tephra-hbase-compat-1.0-cdh/pom.xml
+++ b/tephra-hbase-compat-1.0-cdh/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-hbase-compat-1.0/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/pom.xml b/tephra-hbase-compat-1.0/pom.xml
index c81b708..84e203d 100644
--- a/tephra-hbase-compat-1.0/pom.xml
+++ b/tephra-hbase-compat-1.0/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-hbase-compat-1.1-base/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/pom.xml b/tephra-hbase-compat-1.1-base/pom.xml
index 71a414d..1ae7829 100644
--- a/tephra-hbase-compat-1.1-base/pom.xml
+++ b/tephra-hbase-compat-1.1-base/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
index e6637c5..0ac15be 100644
--- a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
+++ b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.1-base</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
index f36762c..b152305 100644
--- a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
+++ b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.1-base</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/10e36e63/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
index 4f5981f..359af98 100644
--- a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
+++ b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.1-base</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
[5/9] incubator-tephra git commit: (TEPHRA-226) Use older API to run
as login user
Posted by go...@apache.org.
(TEPHRA-226) Use older API to run as login user
This closes #39 from GitHub.
Signed-off-by: Gokul Gunasekaran <go...@cask.co>
Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/95c6bfb6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/95c6bfb6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/95c6bfb6
Branch: refs/heads/master
Commit: 95c6bfb6b4391686745499e19966a87c528b1ddd
Parents: 808ed2e
Author: Gokul Gunasekaran <go...@cask.co>
Authored: Wed Mar 1 14:19:04 2017 -0800
Committer: Gokul Gunasekaran <go...@cask.co>
Committed: Mon Mar 6 13:44:37 2017 -0800
----------------------------------------------------------------------
.../hbase/txprune/PruneUpperBoundWriter.java | 2 +-
.../hbase/txprune/PruneUpperBoundWriter.java | 18 +++++++++++++-----
.../hbase/txprune/PruneUpperBoundWriter.java | 18 +++++++++++++-----
.../hbase/txprune/PruneUpperBoundWriter.java | 18 +++++++++++++-----
4 files changed, 40 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/95c6bfb6/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
index 7e4a0fa..1c26ef1 100644
--- a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
+++ b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
@@ -161,7 +161,7 @@ public class PruneUpperBoundWriter extends AbstractIdleService {
private void handleException(Exception ex) {
LOG.warn("Cannot record prune upper bound for a region to table " +
tableName.getNamespaceAsString() + ":" + tableName.getNameAsString(), ex);
- if (ex instanceof IOException) {
+ if (ex instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/95c6bfb6/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
index 38c1a6f..1c26ef1 100644
--- a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
+++ b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
@@ -22,8 +22,8 @@ import com.google.common.util.concurrent.AbstractIdleService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.security.UserGroupInformation;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
@@ -106,7 +106,7 @@ public class PruneUpperBoundWriter extends AbstractIdleService {
if (now > (lastChecked + pruneFlushInterval)) {
// should flush data
try {
- User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
+ UserGroupInformation.getLoginUser().doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
// Record prune upper bound
@@ -128,9 +128,9 @@ public class PruneUpperBoundWriter extends AbstractIdleService {
return null;
}
});
- } catch (IOException ex) {
- LOG.warn("Cannot record prune upper bound for a region to table " +
- tableName.getNamespaceAsString() + ":" + tableName.getNameAsString(), ex);
+ } catch (IOException | InterruptedException ex) {
+ // Handle any exception that might be thrown during HBase operation
+ handleException(ex);
}
lastChecked = now;
}
@@ -157,4 +157,12 @@ public class PruneUpperBoundWriter extends AbstractIdleService {
Bytes.toStringBinary(regionName), isRunning() ? "alive" : "running"));
}
}
+
+ private void handleException(Exception ex) {
+ LOG.warn("Cannot record prune upper bound for a region to table " +
+ tableName.getNamespaceAsString() + ":" + tableName.getNameAsString(), ex);
+ if (ex instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/95c6bfb6/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
index 6bd8bab..1c26ef1 100644
--- a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
+++ b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
@@ -22,8 +22,8 @@ import com.google.common.util.concurrent.AbstractIdleService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.security.UserGroupInformation;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
@@ -106,7 +106,7 @@ public class PruneUpperBoundWriter extends AbstractIdleService {
if (now > (lastChecked + pruneFlushInterval)) {
// should flush data
try {
- User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
+ UserGroupInformation.getLoginUser().doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
// Record prune upper bound
@@ -128,9 +128,9 @@ public class PruneUpperBoundWriter extends AbstractIdleService {
return null;
}
});
- } catch (IOException ex) {
- LOG.warn("Cannot record prune upper bound for a region to table " +
- tableName.getNameWithNamespaceInclAsString(), ex);
+ } catch (IOException | InterruptedException ex) {
+ // Handle any exception that might be thrown during HBase operation
+ handleException(ex);
}
lastChecked = now;
}
@@ -157,4 +157,12 @@ public class PruneUpperBoundWriter extends AbstractIdleService {
Bytes.toStringBinary(regionName), isRunning() ? "alive" : "running"));
}
}
+
+ private void handleException(Exception ex) {
+ LOG.warn("Cannot record prune upper bound for a region to table " +
+ tableName.getNamespaceAsString() + ":" + tableName.getNameAsString(), ex);
+ if (ex instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/95c6bfb6/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
index 38c1a6f..1c26ef1 100644
--- a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
+++ b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/txprune/PruneUpperBoundWriter.java
@@ -22,8 +22,8 @@ import com.google.common.util.concurrent.AbstractIdleService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.security.UserGroupInformation;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
@@ -106,7 +106,7 @@ public class PruneUpperBoundWriter extends AbstractIdleService {
if (now > (lastChecked + pruneFlushInterval)) {
// should flush data
try {
- User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
+ UserGroupInformation.getLoginUser().doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
// Record prune upper bound
@@ -128,9 +128,9 @@ public class PruneUpperBoundWriter extends AbstractIdleService {
return null;
}
});
- } catch (IOException ex) {
- LOG.warn("Cannot record prune upper bound for a region to table " +
- tableName.getNamespaceAsString() + ":" + tableName.getNameAsString(), ex);
+ } catch (IOException | InterruptedException ex) {
+ // Handle any exception that might be thrown during HBase operation
+ handleException(ex);
}
lastChecked = now;
}
@@ -157,4 +157,12 @@ public class PruneUpperBoundWriter extends AbstractIdleService {
Bytes.toStringBinary(regionName), isRunning() ? "alive" : "running"));
}
}
+
+ private void handleException(Exception ex) {
+ LOG.warn("Cannot record prune upper bound for a region to table " +
+ tableName.getNamespaceAsString() + ":" + tableName.getNameAsString(), ex);
+ if (ex instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ }
}
[4/9] incubator-tephra git commit: TEPHRA-224 Handle delay between
transaction max lifetime check and data writes while pruning
Posted by go...@apache.org.
TEPHRA-224 Handle delay between transaction max lifetime check and data writes while pruning
This closes #38
Signed-off-by: poorna <po...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/808ed2e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/808ed2e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/808ed2e3
Branch: refs/heads/master
Commit: 808ed2e3fe40c86cf05442e0d842bbc844ddd857
Parents: 872fb10
Author: poorna <po...@cask.co>
Authored: Tue Feb 21 17:15:20 2017 -0800
Committer: poorna <po...@apache.org>
Committed: Wed Feb 22 14:49:16 2017 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/tephra/TxConstants.java | 6 ++++++
.../tephra/txprune/TransactionPruningRunnable.java | 12 ++++++++++--
.../tephra/txprune/TransactionPruningService.java | 9 ++++++---
.../tephra/txprune/TransactionPruningServiceTest.java | 10 ++++++----
4 files changed, 28 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/808ed2e3/tephra-core/src/main/java/org/apache/tephra/TxConstants.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/TxConstants.java b/tephra-core/src/main/java/org/apache/tephra/TxConstants.java
index ebf91e3..26a48fb 100644
--- a/tephra-core/src/main/java/org/apache/tephra/TxConstants.java
+++ b/tephra-core/src/main/java/org/apache/tephra/TxConstants.java
@@ -376,6 +376,11 @@ public class TxConstants {
public static final String PRUNE_FLUSH_INTERVAL = "data.tx.prune.flush.interval";
/**
+ * The time in seconds used to pad transaction max lifetime while pruning.
+ */
+ public static final String PRUNE_GRACE_PERIOD = "data.tx.grace.period";
+
+ /**
* Comma separated list of invalid transaction pruning plugins to load
*/
public static final String PLUGINS = "data.tx.prune.plugins";
@@ -388,6 +393,7 @@ public class TxConstants {
public static final String DEFAULT_PRUNE_STATE_TABLE = "tephra.state";
public static final long DEFAULT_PRUNE_INTERVAL = TimeUnit.HOURS.toSeconds(6);
public static final long DEFAULT_PRUNE_FLUSH_INTERVAL = TimeUnit.MINUTES.toSeconds(1);
+ public static final long DEFAULT_PRUNE_GRACE_PERIOD = TimeUnit.HOURS.toSeconds(24);
public static final String DEFAULT_PLUGIN = "data.tx.prune.plugin.default";
public static final String DEFAULT_PLUGIN_CLASS =
"org.apache.tephra.hbase.txprune.HBaseTransactionPruningPlugin";
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/808ed2e3/tephra-core/src/main/java/org/apache/tephra/txprune/TransactionPruningRunnable.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/txprune/TransactionPruningRunnable.java b/tephra-core/src/main/java/org/apache/tephra/txprune/TransactionPruningRunnable.java
index d73c50a..89ed25e 100644
--- a/tephra-core/src/main/java/org/apache/tephra/txprune/TransactionPruningRunnable.java
+++ b/tephra-core/src/main/java/org/apache/tephra/txprune/TransactionPruningRunnable.java
@@ -36,18 +36,21 @@ import java.util.TreeSet;
* This class executes one run of transaction pruning every time it is invoked.
* Typically, this class will be scheduled to run periodically.
*/
+@SuppressWarnings("WeakerAccess")
public class TransactionPruningRunnable implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(TransactionPruningRunnable.class);
private final TransactionManager txManager;
private final Map<String, TransactionPruningPlugin> plugins;
private final long txMaxLifetimeMillis;
+ private final long txPruneBufferMillis;
public TransactionPruningRunnable(TransactionManager txManager, Map<String, TransactionPruningPlugin> plugins,
- long txMaxLifetimeMillis) {
+ long txMaxLifetimeMillis, long txPruneBufferMillis) {
this.txManager = txManager;
this.plugins = plugins;
this.txMaxLifetimeMillis = txMaxLifetimeMillis;
+ this.txPruneBufferMillis = txPruneBufferMillis;
}
@Override
@@ -57,8 +60,13 @@ public class TransactionPruningRunnable implements Runnable {
Transaction tx = txManager.startShort();
txManager.abort(tx);
+ if (tx.getInvalids().length == 0) {
+ LOG.info("Invalid list is empty, not running transaction pruning");
+ return;
+ }
+
long now = getTime();
- long inactiveTransactionBound = TxUtils.getInactiveTxBound(now, txMaxLifetimeMillis);
+ long inactiveTransactionBound = TxUtils.getInactiveTxBound(now, txMaxLifetimeMillis + txPruneBufferMillis);
LOG.info("Starting invalid prune run for time {} and inactive transaction bound {}",
now, inactiveTransactionBound);
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/808ed2e3/tephra-core/src/main/java/org/apache/tephra/txprune/TransactionPruningService.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/txprune/TransactionPruningService.java b/tephra-core/src/main/java/org/apache/tephra/txprune/TransactionPruningService.java
index d80bbd4..8d7fe2f 100644
--- a/tephra-core/src/main/java/org/apache/tephra/txprune/TransactionPruningService.java
+++ b/tephra-core/src/main/java/org/apache/tephra/txprune/TransactionPruningService.java
@@ -78,8 +78,11 @@ public class TransactionPruningService extends AbstractIdleService {
Map<String, TransactionPruningPlugin> plugins = initializePlugins();
long txMaxLifetimeMillis = TimeUnit.SECONDS.toMillis(conf.getInt(TxConstants.Manager.CFG_TX_MAX_LIFETIME,
TxConstants.Manager.DEFAULT_TX_MAX_LIFETIME));
+ long txPruneBufferMillis =
+ TimeUnit.SECONDS.toMillis(conf.getLong(TxConstants.TransactionPruning.PRUNE_GRACE_PERIOD,
+ TxConstants.TransactionPruning.DEFAULT_PRUNE_GRACE_PERIOD));
scheduledExecutorService.scheduleAtFixedRate(
- getTxPruneRunnable(txManager, plugins, txMaxLifetimeMillis),
+ getTxPruneRunnable(txManager, plugins, txMaxLifetimeMillis, txPruneBufferMillis),
scheduleInterval, scheduleInterval, TimeUnit.SECONDS);
LOG.info("Scheduled {} plugins with interval {} seconds", plugins.size(), scheduleInterval);
}
@@ -104,8 +107,8 @@ public class TransactionPruningService extends AbstractIdleService {
@VisibleForTesting
TransactionPruningRunnable getTxPruneRunnable(TransactionManager txManager,
Map<String, TransactionPruningPlugin> plugins,
- long txMaxLifetimeMillis) {
- return new TransactionPruningRunnable(txManager, plugins, txMaxLifetimeMillis);
+ long txMaxLifetimeMillis, long txPruneBufferMillis) {
+ return new TransactionPruningRunnable(txManager, plugins, txMaxLifetimeMillis, txPruneBufferMillis);
}
private Map<String, TransactionPruningPlugin> initializePlugins()
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/808ed2e3/tephra-core/src/test/java/org/apache/tephra/txprune/TransactionPruningServiceTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/txprune/TransactionPruningServiceTest.java b/tephra-core/src/test/java/org/apache/tephra/txprune/TransactionPruningServiceTest.java
index 9c23ab7..2a0a17e 100644
--- a/tephra-core/src/test/java/org/apache/tephra/txprune/TransactionPruningServiceTest.java
+++ b/tephra-core/src/test/java/org/apache/tephra/txprune/TransactionPruningServiceTest.java
@@ -69,6 +69,7 @@ public class TransactionPruningServiceTest {
conf.setBoolean(TxConstants.TransactionPruning.PRUNE_ENABLE, true);
conf.setInt(TxConstants.TransactionPruning.PRUNE_INTERVAL, 1);
conf.setInt(TxConstants.Manager.CFG_TX_MAX_LIFETIME, 10);
+ conf.setLong(TxConstants.TransactionPruning.PRUNE_GRACE_PERIOD, 0);
// Setup mock data
long m = 1000;
@@ -132,6 +133,7 @@ public class TransactionPruningServiceTest {
conf.setBoolean(TxConstants.TransactionPruning.PRUNE_ENABLE, true);
conf.setInt(TxConstants.TransactionPruning.PRUNE_INTERVAL, 1);
conf.setInt(TxConstants.Manager.CFG_TX_MAX_LIFETIME, 10);
+ conf.setLong(TxConstants.TransactionPruning.PRUNE_GRACE_PERIOD, 0);
// Setup mock data
long m = 1000;
@@ -222,8 +224,8 @@ public class TransactionPruningServiceTest {
@Override
TransactionPruningRunnable getTxPruneRunnable(TransactionManager txManager,
Map<String, TransactionPruningPlugin> plugins,
- long txMaxLifetimeMillis) {
- return new TestTransactionPruningRunnable(txManager, plugins, txMaxLifetimeMillis);
+ long txMaxLifetimeMillis, long txPruneBufferMillis) {
+ return new TestTransactionPruningRunnable(txManager, plugins, txMaxLifetimeMillis, txPruneBufferMillis);
}
}
@@ -233,8 +235,8 @@ public class TransactionPruningServiceTest {
private static class TestTransactionPruningRunnable extends TransactionPruningRunnable {
private static Iterator<Long> currentTime;
TestTransactionPruningRunnable(TransactionManager txManager, Map<String, TransactionPruningPlugin> plugins,
- long txMaxLifetimeMillis) {
- super(txManager, plugins, txMaxLifetimeMillis);
+ long txMaxLifetimeMillis, long txPruneBufferMillis) {
+ super(txManager, plugins, txMaxLifetimeMillis, txPruneBufferMillis);
}
@Override
[6/9] incubator-tephra git commit: (TEPHRA-227) Add new command to
get the set of regions that have not been compacted
Posted by go...@apache.org.
(TEPHRA-227) Add new command to get the set of regions that have not been compacted
This closes #40 from GitHub.
Signed-off-by: Gokul Gunasekaran <go...@cask.co>
Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/66a2fce7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/66a2fce7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/66a2fce7
Branch: refs/heads/master
Commit: 66a2fce795dec34da7b992aeda883345fbbf3083
Parents: 95c6bfb
Author: Gokul Gunasekaran <go...@cask.co>
Authored: Wed Mar 1 14:54:40 2017 -0800
Committer: Gokul Gunasekaran <go...@cask.co>
Committed: Mon Mar 6 13:47:14 2017 -0800
----------------------------------------------------------------------
.../hbase/txprune/InvalidListPruningDebug.java | 105 +++++++++++++++++--
.../hbase/txprune/InvalidListPruningDebug.java | 105 +++++++++++++++++--
.../hbase/txprune/InvalidListPruningDebug.java | 105 +++++++++++++++++--
.../hbase/txprune/InvalidListPruningDebug.java | 105 +++++++++++++++++--
.../hbase/txprune/InvalidListPruningDebug.java | 105 +++++++++++++++++--
5 files changed, 495 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/66a2fce7/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
index d48e48d..620885b 100644
--- a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
+++ b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
@@ -21,6 +21,7 @@ package org.apache.tephra.hbase.txprune;
import com.google.common.collect.Iterables;
import com.google.common.collect.MinMaxPriorityQueue;
+import com.google.common.collect.Sets;
import com.google.gson.Gson;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -36,12 +37,15 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.PrintWriter;
+import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
+import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import javax.annotation.Nullable;
@@ -53,6 +57,8 @@ public class InvalidListPruningDebug {
private static final Logger LOG = LoggerFactory.getLogger(InvalidListPruningDebug.class);
private static final Gson GSON = new Gson();
private DataJanitorState dataJanitorState;
+ private HConnection connection;
+ private TableName tableName;
/**
* Initialize the Invalid List Debug Tool.
@@ -61,20 +67,76 @@ public class InvalidListPruningDebug {
*/
public void initialize(final Configuration conf) throws IOException {
LOG.debug("InvalidListPruningDebugMain : initialize method called");
- final HConnection connection = new HBaseAdmin(conf).getConnection();
+ connection = new HBaseAdmin(conf).getConnection();
+ tableName = TableName.valueOf(conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
+ TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE));
dataJanitorState = new DataJanitorState(new DataJanitorState.TableSupplier() {
@Override
public HTableInterface get() throws IOException {
- return connection.getTable(TableName.valueOf(
- conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
- TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE)));
+ return connection.getTable(tableName);
}
});
}
+ public void destroy() throws IOException {
+ if (connection != null) {
+ connection.close();
+ }
+ }
+
+ /**
+ * Returns a set of regions that are live but are not empty nor have a prune upper bound recorded. These regions
+ * will stop the progress of pruning.
+ *
+ * @param numRegions number of regions
+ * @return {@link Set} of regions that needs to be compacted and flushed
+ */
+ public Set<String> getRegionsToBeCompacted(Integer numRegions) throws IOException {
+ // Fetch the live regions
+ Map<Long, SortedSet<String>> latestTimeRegion = getRegionsOnOrBeforeTime(System.currentTimeMillis());
+ if (latestTimeRegion.isEmpty()) {
+ return new HashSet<>();
+ }
+
+ Long timestamp = latestTimeRegion.keySet().iterator().next();
+ SortedSet<String> liveRegions = latestTimeRegion.get(timestamp);
+
+ SortedSet<byte[]> emptyRegions = dataJanitorState.getEmptyRegionsAfterTime(timestamp, null);
+ SortedSet<String> emptyRegionNames = new TreeSet<>();
+ Iterable<String> regionStrings = Iterables.transform(emptyRegions, TimeRegions.BYTE_ARR_TO_STRING_FN);
+ for (String regionString : regionStrings) {
+ emptyRegionNames.add(regionString);
+ }
+
+ Set<String> nonEmptyRegions = Sets.newHashSet(Sets.difference(liveRegions, emptyRegionNames));
+
+ // Get all pruned regions and remove them from the nonEmptyRegions, resulting in a set of regions that are
+ // not empty and have not been registered prune upper bound
+ Queue<RegionPruneInfo> prunedRegions = getIdleRegions(-1);
+ for (RegionPruneInfo prunedRegion : prunedRegions) {
+ if (nonEmptyRegions.contains(prunedRegion.getRegionNameAsString())) {
+ nonEmptyRegions.remove(prunedRegion.getRegionNameAsString());
+ }
+ }
+
+ if ((numRegions < 0) || (numRegions >= nonEmptyRegions.size())) {
+ return nonEmptyRegions;
+ }
+
+ Set<String> subsetRegions = new HashSet<>(numRegions);
+ for (String regionName : nonEmptyRegions) {
+ if (subsetRegions.size() == numRegions) {
+ break;
+ }
+ subsetRegions.add(regionName);
+ }
+ return subsetRegions;
+ }
+
/**
* Return a list of RegionPruneInfo. These regions are the ones that have the lowest prune upper bounds.
- * If -1 is passed in, all the regions and their prune upper bound will be returned.
+ * If -1 is passed in, all the regions and their prune upper bound will be returned. Note that only the regions
+ * that are known to be live will be returned.
*
* @param numRegions number of regions
* @return Map of region name and its prune upper bound
@@ -85,10 +147,32 @@ public class InvalidListPruningDebug {
return new LinkedList<>();
}
+ // Create a set with region names
+ Set<String> pruneRegionNameSet = new HashSet<>();
+ for (RegionPruneInfo regionPruneInfo : regionPruneInfos) {
+ pruneRegionNameSet.add(regionPruneInfo.getRegionNameAsString());
+ }
+
+ // Fetch the live regions
+ Map<Long, SortedSet<String>> latestTimeRegion = getRegionsOnOrBeforeTime(System.currentTimeMillis());
+ if (!latestTimeRegion.isEmpty()) {
+ SortedSet<String> liveRegions = latestTimeRegion.values().iterator().next();
+ Set<String> liveRegionsWithPruneInfo = Sets.intersection(liveRegions, pruneRegionNameSet);
+ List<RegionPruneInfo> liveRegionWithPruneInfoList = new ArrayList<>();
+ for (RegionPruneInfo regionPruneInfo : regionPruneInfos) {
+ if (liveRegionsWithPruneInfo.contains(regionPruneInfo.getRegionNameAsString())) {
+ liveRegionWithPruneInfoList.add(regionPruneInfo);
+ }
+ }
+
+ // Use the subset of live regions and prune regions
+ regionPruneInfos = liveRegionWithPruneInfoList;
+ }
+
if (numRegions < 0) {
numRegions = regionPruneInfos.size();
}
-
+
Queue<RegionPruneInfo> lowestPrunes = MinMaxPriorityQueue.orderedBy(new Comparator<RegionPruneInfo>() {
@Override
public int compare(RegionPruneInfo o1, RegionPruneInfo o2) {
@@ -148,6 +232,9 @@ public class InvalidListPruningDebug {
"provided as the limit, prune upper bounds of all regions are returned.");
pw.println("prune-info region-name-as-string");
pw.println("Desc: Prints out the Pruning information for the region 'region-name-as-string'");
+ pw.println("to-compact-regions limit");
+ pw.println("Desc: Prints out 'limit' number of regions that are active, but are not empty, " +
+ "and have not registered a prune upper bound.");
}
private boolean execute(String[] args) throws IOException {
@@ -177,6 +264,11 @@ public class InvalidListPruningDebug {
pw.println(String.format("No prune info found for the region %s.", parameter));
}
return true;
+ } else if ("to-compact-regions".equals(command)) {
+ Integer numRegions = Integer.parseInt(parameter);
+ Set<String> toBeCompactedRegions = getRegionsToBeCompacted(numRegions);
+ pw.println(GSON.toJson(toBeCompactedRegions));
+ return true;
} else {
pw.println(String.format("%s is not a valid command.", command));
printUsage(pw);
@@ -191,6 +283,7 @@ public class InvalidListPruningDebug {
try {
pruningDebug.initialize(hConf);
boolean success = pruningDebug.execute(args);
+ pruningDebug.destroy();
if (!success) {
System.exit(1);
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/66a2fce7/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
index d48e48d..620885b 100644
--- a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
+++ b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
@@ -21,6 +21,7 @@ package org.apache.tephra.hbase.txprune;
import com.google.common.collect.Iterables;
import com.google.common.collect.MinMaxPriorityQueue;
+import com.google.common.collect.Sets;
import com.google.gson.Gson;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -36,12 +37,15 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.PrintWriter;
+import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
+import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import javax.annotation.Nullable;
@@ -53,6 +57,8 @@ public class InvalidListPruningDebug {
private static final Logger LOG = LoggerFactory.getLogger(InvalidListPruningDebug.class);
private static final Gson GSON = new Gson();
private DataJanitorState dataJanitorState;
+ private HConnection connection;
+ private TableName tableName;
/**
* Initialize the Invalid List Debug Tool.
@@ -61,20 +67,76 @@ public class InvalidListPruningDebug {
*/
public void initialize(final Configuration conf) throws IOException {
LOG.debug("InvalidListPruningDebugMain : initialize method called");
- final HConnection connection = new HBaseAdmin(conf).getConnection();
+ connection = new HBaseAdmin(conf).getConnection();
+ tableName = TableName.valueOf(conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
+ TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE));
dataJanitorState = new DataJanitorState(new DataJanitorState.TableSupplier() {
@Override
public HTableInterface get() throws IOException {
- return connection.getTable(TableName.valueOf(
- conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
- TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE)));
+ return connection.getTable(tableName);
}
});
}
+ public void destroy() throws IOException {
+ if (connection != null) {
+ connection.close();
+ }
+ }
+
+ /**
+ * Returns a set of regions that are live but are not empty nor have a prune upper bound recorded. These regions
+ * will stop the progress of pruning.
+ *
+ * @param numRegions number of regions
+ * @return {@link Set} of regions that needs to be compacted and flushed
+ */
+ public Set<String> getRegionsToBeCompacted(Integer numRegions) throws IOException {
+ // Fetch the live regions
+ Map<Long, SortedSet<String>> latestTimeRegion = getRegionsOnOrBeforeTime(System.currentTimeMillis());
+ if (latestTimeRegion.isEmpty()) {
+ return new HashSet<>();
+ }
+
+ Long timestamp = latestTimeRegion.keySet().iterator().next();
+ SortedSet<String> liveRegions = latestTimeRegion.get(timestamp);
+
+ SortedSet<byte[]> emptyRegions = dataJanitorState.getEmptyRegionsAfterTime(timestamp, null);
+ SortedSet<String> emptyRegionNames = new TreeSet<>();
+ Iterable<String> regionStrings = Iterables.transform(emptyRegions, TimeRegions.BYTE_ARR_TO_STRING_FN);
+ for (String regionString : regionStrings) {
+ emptyRegionNames.add(regionString);
+ }
+
+ Set<String> nonEmptyRegions = Sets.newHashSet(Sets.difference(liveRegions, emptyRegionNames));
+
+ // Get all pruned regions and remove them from the nonEmptyRegions, resulting in a set of regions that are
+ // not empty and have not been registered prune upper bound
+ Queue<RegionPruneInfo> prunedRegions = getIdleRegions(-1);
+ for (RegionPruneInfo prunedRegion : prunedRegions) {
+ if (nonEmptyRegions.contains(prunedRegion.getRegionNameAsString())) {
+ nonEmptyRegions.remove(prunedRegion.getRegionNameAsString());
+ }
+ }
+
+ if ((numRegions < 0) || (numRegions >= nonEmptyRegions.size())) {
+ return nonEmptyRegions;
+ }
+
+ Set<String> subsetRegions = new HashSet<>(numRegions);
+ for (String regionName : nonEmptyRegions) {
+ if (subsetRegions.size() == numRegions) {
+ break;
+ }
+ subsetRegions.add(regionName);
+ }
+ return subsetRegions;
+ }
+
/**
* Return a list of RegionPruneInfo. These regions are the ones that have the lowest prune upper bounds.
- * If -1 is passed in, all the regions and their prune upper bound will be returned.
+ * If -1 is passed in, all the regions and their prune upper bound will be returned. Note that only the regions
+ * that are known to be live will be returned.
*
* @param numRegions number of regions
* @return Map of region name and its prune upper bound
@@ -85,10 +147,32 @@ public class InvalidListPruningDebug {
return new LinkedList<>();
}
+ // Create a set with region names
+ Set<String> pruneRegionNameSet = new HashSet<>();
+ for (RegionPruneInfo regionPruneInfo : regionPruneInfos) {
+ pruneRegionNameSet.add(regionPruneInfo.getRegionNameAsString());
+ }
+
+ // Fetch the live regions
+ Map<Long, SortedSet<String>> latestTimeRegion = getRegionsOnOrBeforeTime(System.currentTimeMillis());
+ if (!latestTimeRegion.isEmpty()) {
+ SortedSet<String> liveRegions = latestTimeRegion.values().iterator().next();
+ Set<String> liveRegionsWithPruneInfo = Sets.intersection(liveRegions, pruneRegionNameSet);
+ List<RegionPruneInfo> liveRegionWithPruneInfoList = new ArrayList<>();
+ for (RegionPruneInfo regionPruneInfo : regionPruneInfos) {
+ if (liveRegionsWithPruneInfo.contains(regionPruneInfo.getRegionNameAsString())) {
+ liveRegionWithPruneInfoList.add(regionPruneInfo);
+ }
+ }
+
+ // Use the subset of live regions and prune regions
+ regionPruneInfos = liveRegionWithPruneInfoList;
+ }
+
if (numRegions < 0) {
numRegions = regionPruneInfos.size();
}
-
+
Queue<RegionPruneInfo> lowestPrunes = MinMaxPriorityQueue.orderedBy(new Comparator<RegionPruneInfo>() {
@Override
public int compare(RegionPruneInfo o1, RegionPruneInfo o2) {
@@ -148,6 +232,9 @@ public class InvalidListPruningDebug {
"provided as the limit, prune upper bounds of all regions are returned.");
pw.println("prune-info region-name-as-string");
pw.println("Desc: Prints out the Pruning information for the region 'region-name-as-string'");
+ pw.println("to-compact-regions limit");
+ pw.println("Desc: Prints out 'limit' number of regions that are active, but are not empty, " +
+ "and have not registered a prune upper bound.");
}
private boolean execute(String[] args) throws IOException {
@@ -177,6 +264,11 @@ public class InvalidListPruningDebug {
pw.println(String.format("No prune info found for the region %s.", parameter));
}
return true;
+ } else if ("to-compact-regions".equals(command)) {
+ Integer numRegions = Integer.parseInt(parameter);
+ Set<String> toBeCompactedRegions = getRegionsToBeCompacted(numRegions);
+ pw.println(GSON.toJson(toBeCompactedRegions));
+ return true;
} else {
pw.println(String.format("%s is not a valid command.", command));
printUsage(pw);
@@ -191,6 +283,7 @@ public class InvalidListPruningDebug {
try {
pruningDebug.initialize(hConf);
boolean success = pruningDebug.execute(args);
+ pruningDebug.destroy();
if (!success) {
System.exit(1);
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/66a2fce7/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
index e748d90..443c998 100644
--- a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
+++ b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
@@ -21,6 +21,7 @@ package org.apache.tephra.hbase.txprune;
import com.google.common.collect.Iterables;
import com.google.common.collect.MinMaxPriorityQueue;
+import com.google.common.collect.Sets;
import com.google.gson.Gson;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -36,12 +37,15 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.PrintWriter;
+import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
+import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import javax.annotation.Nullable;
@@ -53,6 +57,8 @@ public class InvalidListPruningDebug {
private static final Logger LOG = LoggerFactory.getLogger(InvalidListPruningDebug.class);
private static final Gson GSON = new Gson();
private DataJanitorState dataJanitorState;
+ private Connection connection;
+ private TableName tableName;
/**
* Initialize the Invalid List Debug Tool.
@@ -61,20 +67,76 @@ public class InvalidListPruningDebug {
*/
public void initialize(final Configuration conf) throws IOException {
LOG.debug("InvalidListPruningDebugMain : initialize method called");
- final Connection connection = ConnectionFactory.createConnection(conf);
+ connection = ConnectionFactory.createConnection(conf);
+ tableName = TableName.valueOf(conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
+ TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE));
dataJanitorState = new DataJanitorState(new DataJanitorState.TableSupplier() {
@Override
public Table get() throws IOException {
- return connection.getTable(TableName.valueOf(
- conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
- TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE)));
+ return connection.getTable(tableName);
}
});
}
+ public void destroy() throws IOException {
+ if (connection != null) {
+ connection.close();
+ }
+ }
+
+ /**
+ * Returns a set of regions that are live but are not empty nor have a prune upper bound recorded. These regions
+ * will stop the progress of pruning.
+ *
+ * @param numRegions number of regions
+ * @return {@link Set} of regions that needs to be compacted and flushed
+ */
+ public Set<String> getRegionsToBeCompacted(Integer numRegions) throws IOException {
+ // Fetch the live regions
+ Map<Long, SortedSet<String>> latestTimeRegion = getRegionsOnOrBeforeTime(System.currentTimeMillis());
+ if (latestTimeRegion.isEmpty()) {
+ return new HashSet<>();
+ }
+
+ Long timestamp = latestTimeRegion.keySet().iterator().next();
+ SortedSet<String> liveRegions = latestTimeRegion.get(timestamp);
+
+ SortedSet<byte[]> emptyRegions = dataJanitorState.getEmptyRegionsAfterTime(timestamp, null);
+ SortedSet<String> emptyRegionNames = new TreeSet<>();
+ Iterable<String> regionStrings = Iterables.transform(emptyRegions, TimeRegions.BYTE_ARR_TO_STRING_FN);
+ for (String regionString : regionStrings) {
+ emptyRegionNames.add(regionString);
+ }
+
+ Set<String> nonEmptyRegions = Sets.newHashSet(Sets.difference(liveRegions, emptyRegionNames));
+
+ // Get all pruned regions and remove them from the nonEmptyRegions, resulting in a set of regions that are
+ // not empty and have not been registered prune upper bound
+ Queue<RegionPruneInfo> prunedRegions = getIdleRegions(-1);
+ for (RegionPruneInfo prunedRegion : prunedRegions) {
+ if (nonEmptyRegions.contains(prunedRegion.getRegionNameAsString())) {
+ nonEmptyRegions.remove(prunedRegion.getRegionNameAsString());
+ }
+ }
+
+ if ((numRegions < 0) || (numRegions >= nonEmptyRegions.size())) {
+ return nonEmptyRegions;
+ }
+
+ Set<String> subsetRegions = new HashSet<>(numRegions);
+ for (String regionName : nonEmptyRegions) {
+ if (subsetRegions.size() == numRegions) {
+ break;
+ }
+ subsetRegions.add(regionName);
+ }
+ return subsetRegions;
+ }
+
/**
* Return a list of RegionPruneInfo. These regions are the ones that have the lowest prune upper bounds.
- * If -1 is passed in, all the regions and their prune upper bound will be returned.
+ * If -1 is passed in, all the regions and their prune upper bound will be returned. Note that only the regions
+ * that are known to be live will be returned.
*
* @param numRegions number of regions
* @return Map of region name and its prune upper bound
@@ -85,10 +147,32 @@ public class InvalidListPruningDebug {
return new LinkedList<>();
}
+ // Create a set with region names
+ Set<String> pruneRegionNameSet = new HashSet<>();
+ for (RegionPruneInfo regionPruneInfo : regionPruneInfos) {
+ pruneRegionNameSet.add(regionPruneInfo.getRegionNameAsString());
+ }
+
+ // Fetch the live regions
+ Map<Long, SortedSet<String>> latestTimeRegion = getRegionsOnOrBeforeTime(System.currentTimeMillis());
+ if (!latestTimeRegion.isEmpty()) {
+ SortedSet<String> liveRegions = latestTimeRegion.values().iterator().next();
+ Set<String> liveRegionsWithPruneInfo = Sets.intersection(liveRegions, pruneRegionNameSet);
+ List<RegionPruneInfo> liveRegionWithPruneInfoList = new ArrayList<>();
+ for (RegionPruneInfo regionPruneInfo : regionPruneInfos) {
+ if (liveRegionsWithPruneInfo.contains(regionPruneInfo.getRegionNameAsString())) {
+ liveRegionWithPruneInfoList.add(regionPruneInfo);
+ }
+ }
+
+ // Use the subset of live regions and prune regions
+ regionPruneInfos = liveRegionWithPruneInfoList;
+ }
+
if (numRegions < 0) {
numRegions = regionPruneInfos.size();
}
-
+
Queue<RegionPruneInfo> lowestPrunes = MinMaxPriorityQueue.orderedBy(new Comparator<RegionPruneInfo>() {
@Override
public int compare(RegionPruneInfo o1, RegionPruneInfo o2) {
@@ -148,6 +232,9 @@ public class InvalidListPruningDebug {
"provided as the limit, prune upper bounds of all regions are returned.");
pw.println("prune-info region-name-as-string");
pw.println("Desc: Prints out the Pruning information for the region 'region-name-as-string'");
+ pw.println("to-compact-regions limit");
+ pw.println("Desc: Prints out 'limit' number of regions that are active, but are not empty, " +
+ "and have not registered a prune upper bound.");
}
private boolean execute(String[] args) throws IOException {
@@ -177,6 +264,11 @@ public class InvalidListPruningDebug {
pw.println(String.format("No prune info found for the region %s.", parameter));
}
return true;
+ } else if ("to-compact-regions".equals(command)) {
+ Integer numRegions = Integer.parseInt(parameter);
+ Set<String> toBeCompactedRegions = getRegionsToBeCompacted(numRegions);
+ pw.println(GSON.toJson(toBeCompactedRegions));
+ return true;
} else {
pw.println(String.format("%s is not a valid command.", command));
printUsage(pw);
@@ -191,6 +283,7 @@ public class InvalidListPruningDebug {
try {
pruningDebug.initialize(hConf);
boolean success = pruningDebug.execute(args);
+ pruningDebug.destroy();
if (!success) {
System.exit(1);
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/66a2fce7/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
index e748d90..443c998 100644
--- a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
+++ b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
@@ -21,6 +21,7 @@ package org.apache.tephra.hbase.txprune;
import com.google.common.collect.Iterables;
import com.google.common.collect.MinMaxPriorityQueue;
+import com.google.common.collect.Sets;
import com.google.gson.Gson;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -36,12 +37,15 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.PrintWriter;
+import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
+import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import javax.annotation.Nullable;
@@ -53,6 +57,8 @@ public class InvalidListPruningDebug {
private static final Logger LOG = LoggerFactory.getLogger(InvalidListPruningDebug.class);
private static final Gson GSON = new Gson();
private DataJanitorState dataJanitorState;
+ private Connection connection;
+ private TableName tableName;
/**
* Initialize the Invalid List Debug Tool.
@@ -61,20 +67,76 @@ public class InvalidListPruningDebug {
*/
public void initialize(final Configuration conf) throws IOException {
LOG.debug("InvalidListPruningDebugMain : initialize method called");
- final Connection connection = ConnectionFactory.createConnection(conf);
+ connection = ConnectionFactory.createConnection(conf);
+ tableName = TableName.valueOf(conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
+ TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE));
dataJanitorState = new DataJanitorState(new DataJanitorState.TableSupplier() {
@Override
public Table get() throws IOException {
- return connection.getTable(TableName.valueOf(
- conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
- TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE)));
+ return connection.getTable(tableName);
}
});
}
+ public void destroy() throws IOException {
+ if (connection != null) {
+ connection.close();
+ }
+ }
+
+ /**
+ * Returns a set of regions that are live but are not empty nor have a prune upper bound recorded. These regions
+ * will stop the progress of pruning.
+ *
+ * @param numRegions number of regions
+ * @return {@link Set} of regions that needs to be compacted and flushed
+ */
+ public Set<String> getRegionsToBeCompacted(Integer numRegions) throws IOException {
+ // Fetch the live regions
+ Map<Long, SortedSet<String>> latestTimeRegion = getRegionsOnOrBeforeTime(System.currentTimeMillis());
+ if (latestTimeRegion.isEmpty()) {
+ return new HashSet<>();
+ }
+
+ Long timestamp = latestTimeRegion.keySet().iterator().next();
+ SortedSet<String> liveRegions = latestTimeRegion.get(timestamp);
+
+ SortedSet<byte[]> emptyRegions = dataJanitorState.getEmptyRegionsAfterTime(timestamp, null);
+ SortedSet<String> emptyRegionNames = new TreeSet<>();
+ Iterable<String> regionStrings = Iterables.transform(emptyRegions, TimeRegions.BYTE_ARR_TO_STRING_FN);
+ for (String regionString : regionStrings) {
+ emptyRegionNames.add(regionString);
+ }
+
+ Set<String> nonEmptyRegions = Sets.newHashSet(Sets.difference(liveRegions, emptyRegionNames));
+
+ // Get all pruned regions and remove them from the nonEmptyRegions, resulting in a set of regions that are
+ // not empty and have not been registered prune upper bound
+ Queue<RegionPruneInfo> prunedRegions = getIdleRegions(-1);
+ for (RegionPruneInfo prunedRegion : prunedRegions) {
+ if (nonEmptyRegions.contains(prunedRegion.getRegionNameAsString())) {
+ nonEmptyRegions.remove(prunedRegion.getRegionNameAsString());
+ }
+ }
+
+ if ((numRegions < 0) || (numRegions >= nonEmptyRegions.size())) {
+ return nonEmptyRegions;
+ }
+
+ Set<String> subsetRegions = new HashSet<>(numRegions);
+ for (String regionName : nonEmptyRegions) {
+ if (subsetRegions.size() == numRegions) {
+ break;
+ }
+ subsetRegions.add(regionName);
+ }
+ return subsetRegions;
+ }
+
/**
* Return a list of RegionPruneInfo. These regions are the ones that have the lowest prune upper bounds.
- * If -1 is passed in, all the regions and their prune upper bound will be returned.
+ * If -1 is passed in, all the regions and their prune upper bound will be returned. Note that only the regions
+ * that are known to be live will be returned.
*
* @param numRegions number of regions
* @return Map of region name and its prune upper bound
@@ -85,10 +147,32 @@ public class InvalidListPruningDebug {
return new LinkedList<>();
}
+ // Create a set with region names
+ Set<String> pruneRegionNameSet = new HashSet<>();
+ for (RegionPruneInfo regionPruneInfo : regionPruneInfos) {
+ pruneRegionNameSet.add(regionPruneInfo.getRegionNameAsString());
+ }
+
+ // Fetch the live regions
+ Map<Long, SortedSet<String>> latestTimeRegion = getRegionsOnOrBeforeTime(System.currentTimeMillis());
+ if (!latestTimeRegion.isEmpty()) {
+ SortedSet<String> liveRegions = latestTimeRegion.values().iterator().next();
+ Set<String> liveRegionsWithPruneInfo = Sets.intersection(liveRegions, pruneRegionNameSet);
+ List<RegionPruneInfo> liveRegionWithPruneInfoList = new ArrayList<>();
+ for (RegionPruneInfo regionPruneInfo : regionPruneInfos) {
+ if (liveRegionsWithPruneInfo.contains(regionPruneInfo.getRegionNameAsString())) {
+ liveRegionWithPruneInfoList.add(regionPruneInfo);
+ }
+ }
+
+ // Use the subset of live regions and prune regions
+ regionPruneInfos = liveRegionWithPruneInfoList;
+ }
+
if (numRegions < 0) {
numRegions = regionPruneInfos.size();
}
-
+
Queue<RegionPruneInfo> lowestPrunes = MinMaxPriorityQueue.orderedBy(new Comparator<RegionPruneInfo>() {
@Override
public int compare(RegionPruneInfo o1, RegionPruneInfo o2) {
@@ -148,6 +232,9 @@ public class InvalidListPruningDebug {
"provided as the limit, prune upper bounds of all regions are returned.");
pw.println("prune-info region-name-as-string");
pw.println("Desc: Prints out the Pruning information for the region 'region-name-as-string'");
+ pw.println("to-compact-regions limit");
+ pw.println("Desc: Prints out 'limit' number of regions that are active, but are not empty, " +
+ "and have not registered a prune upper bound.");
}
private boolean execute(String[] args) throws IOException {
@@ -177,6 +264,11 @@ public class InvalidListPruningDebug {
pw.println(String.format("No prune info found for the region %s.", parameter));
}
return true;
+ } else if ("to-compact-regions".equals(command)) {
+ Integer numRegions = Integer.parseInt(parameter);
+ Set<String> toBeCompactedRegions = getRegionsToBeCompacted(numRegions);
+ pw.println(GSON.toJson(toBeCompactedRegions));
+ return true;
} else {
pw.println(String.format("%s is not a valid command.", command));
printUsage(pw);
@@ -191,6 +283,7 @@ public class InvalidListPruningDebug {
try {
pruningDebug.initialize(hConf);
boolean success = pruningDebug.execute(args);
+ pruningDebug.destroy();
if (!success) {
System.exit(1);
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/66a2fce7/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java b/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
index e748d90..443c998 100644
--- a/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
+++ b/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java
@@ -21,6 +21,7 @@ package org.apache.tephra.hbase.txprune;
import com.google.common.collect.Iterables;
import com.google.common.collect.MinMaxPriorityQueue;
+import com.google.common.collect.Sets;
import com.google.gson.Gson;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -36,12 +37,15 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.PrintWriter;
+import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
+import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import javax.annotation.Nullable;
@@ -53,6 +57,8 @@ public class InvalidListPruningDebug {
private static final Logger LOG = LoggerFactory.getLogger(InvalidListPruningDebug.class);
private static final Gson GSON = new Gson();
private DataJanitorState dataJanitorState;
+ private Connection connection;
+ private TableName tableName;
/**
* Initialize the Invalid List Debug Tool.
@@ -61,20 +67,76 @@ public class InvalidListPruningDebug {
*/
public void initialize(final Configuration conf) throws IOException {
LOG.debug("InvalidListPruningDebugMain : initialize method called");
- final Connection connection = ConnectionFactory.createConnection(conf);
+ connection = ConnectionFactory.createConnection(conf);
+ tableName = TableName.valueOf(conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
+ TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE));
dataJanitorState = new DataJanitorState(new DataJanitorState.TableSupplier() {
@Override
public Table get() throws IOException {
- return connection.getTable(TableName.valueOf(
- conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
- TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE)));
+ return connection.getTable(tableName);
}
});
}
+ public void destroy() throws IOException {
+ if (connection != null) {
+ connection.close();
+ }
+ }
+
+ /**
+ * Returns a set of regions that are live but are not empty nor have a prune upper bound recorded. These regions
+ * will stop the progress of pruning.
+ *
+ * @param numRegions number of regions
+ * @return {@link Set} of regions that needs to be compacted and flushed
+ */
+ public Set<String> getRegionsToBeCompacted(Integer numRegions) throws IOException {
+ // Fetch the live regions
+ Map<Long, SortedSet<String>> latestTimeRegion = getRegionsOnOrBeforeTime(System.currentTimeMillis());
+ if (latestTimeRegion.isEmpty()) {
+ return new HashSet<>();
+ }
+
+ Long timestamp = latestTimeRegion.keySet().iterator().next();
+ SortedSet<String> liveRegions = latestTimeRegion.get(timestamp);
+
+ SortedSet<byte[]> emptyRegions = dataJanitorState.getEmptyRegionsAfterTime(timestamp, null);
+ SortedSet<String> emptyRegionNames = new TreeSet<>();
+ Iterable<String> regionStrings = Iterables.transform(emptyRegions, TimeRegions.BYTE_ARR_TO_STRING_FN);
+ for (String regionString : regionStrings) {
+ emptyRegionNames.add(regionString);
+ }
+
+ Set<String> nonEmptyRegions = Sets.newHashSet(Sets.difference(liveRegions, emptyRegionNames));
+
+ // Get all pruned regions and remove them from the nonEmptyRegions, resulting in a set of regions that are
+ // not empty and have not been registered prune upper bound
+ Queue<RegionPruneInfo> prunedRegions = getIdleRegions(-1);
+ for (RegionPruneInfo prunedRegion : prunedRegions) {
+ if (nonEmptyRegions.contains(prunedRegion.getRegionNameAsString())) {
+ nonEmptyRegions.remove(prunedRegion.getRegionNameAsString());
+ }
+ }
+
+ if ((numRegions < 0) || (numRegions >= nonEmptyRegions.size())) {
+ return nonEmptyRegions;
+ }
+
+ Set<String> subsetRegions = new HashSet<>(numRegions);
+ for (String regionName : nonEmptyRegions) {
+ if (subsetRegions.size() == numRegions) {
+ break;
+ }
+ subsetRegions.add(regionName);
+ }
+ return subsetRegions;
+ }
+
/**
* Return a list of RegionPruneInfo. These regions are the ones that have the lowest prune upper bounds.
- * If -1 is passed in, all the regions and their prune upper bound will be returned.
+ * If -1 is passed in, all the regions and their prune upper bound will be returned. Note that only the regions
+ * that are known to be live will be returned.
*
* @param numRegions number of regions
* @return Map of region name and its prune upper bound
@@ -85,10 +147,32 @@ public class InvalidListPruningDebug {
return new LinkedList<>();
}
+ // Create a set with region names
+ Set<String> pruneRegionNameSet = new HashSet<>();
+ for (RegionPruneInfo regionPruneInfo : regionPruneInfos) {
+ pruneRegionNameSet.add(regionPruneInfo.getRegionNameAsString());
+ }
+
+ // Fetch the live regions
+ Map<Long, SortedSet<String>> latestTimeRegion = getRegionsOnOrBeforeTime(System.currentTimeMillis());
+ if (!latestTimeRegion.isEmpty()) {
+ SortedSet<String> liveRegions = latestTimeRegion.values().iterator().next();
+ Set<String> liveRegionsWithPruneInfo = Sets.intersection(liveRegions, pruneRegionNameSet);
+ List<RegionPruneInfo> liveRegionWithPruneInfoList = new ArrayList<>();
+ for (RegionPruneInfo regionPruneInfo : regionPruneInfos) {
+ if (liveRegionsWithPruneInfo.contains(regionPruneInfo.getRegionNameAsString())) {
+ liveRegionWithPruneInfoList.add(regionPruneInfo);
+ }
+ }
+
+ // Use the subset of live regions and prune regions
+ regionPruneInfos = liveRegionWithPruneInfoList;
+ }
+
if (numRegions < 0) {
numRegions = regionPruneInfos.size();
}
-
+
Queue<RegionPruneInfo> lowestPrunes = MinMaxPriorityQueue.orderedBy(new Comparator<RegionPruneInfo>() {
@Override
public int compare(RegionPruneInfo o1, RegionPruneInfo o2) {
@@ -148,6 +232,9 @@ public class InvalidListPruningDebug {
"provided as the limit, prune upper bounds of all regions are returned.");
pw.println("prune-info region-name-as-string");
pw.println("Desc: Prints out the Pruning information for the region 'region-name-as-string'");
+ pw.println("to-compact-regions limit");
+ pw.println("Desc: Prints out 'limit' number of regions that are active, but are not empty, " +
+ "and have not registered a prune upper bound.");
}
private boolean execute(String[] args) throws IOException {
@@ -177,6 +264,11 @@ public class InvalidListPruningDebug {
pw.println(String.format("No prune info found for the region %s.", parameter));
}
return true;
+ } else if ("to-compact-regions".equals(command)) {
+ Integer numRegions = Integer.parseInt(parameter);
+ Set<String> toBeCompactedRegions = getRegionsToBeCompacted(numRegions);
+ pw.println(GSON.toJson(toBeCompactedRegions));
+ return true;
} else {
pw.println(String.format("%s is not a valid command.", command));
printUsage(pw);
@@ -191,6 +283,7 @@ public class InvalidListPruningDebug {
try {
pruningDebug.initialize(hConf);
boolean success = pruningDebug.execute(args);
+ pruningDebug.destroy();
if (!success) {
System.exit(1);
}
[8/9] incubator-tephra git commit: [maven-release-plugin] prepare for
next development iteration
Posted by go...@apache.org.
[maven-release-plugin] prepare for next development iteration
Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/14aca68a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/14aca68a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/14aca68a
Branch: refs/heads/master
Commit: 14aca68adc9adf3590ae943b13d3b229934ec22c
Parents: a12cff8
Author: Gokul Gunasekaran <go...@cask.co>
Authored: Mon Mar 6 15:35:54 2017 -0800
Committer: Gokul Gunasekaran <go...@cask.co>
Committed: Mon Mar 6 15:35:54 2017 -0800
----------------------------------------------------------------------
pom.xml | 4 ++--
tephra-api/pom.xml | 2 +-
tephra-core/pom.xml | 2 +-
tephra-distribution/pom.xml | 2 +-
tephra-examples/cdh-5.7/pom.xml | 2 +-
tephra-examples/cdh-5.8/pom.xml | 2 +-
tephra-examples/hbase-0.96/pom.xml | 2 +-
tephra-examples/hbase-0.98/pom.xml | 2 +-
tephra-examples/hbase-1.0-cdh/pom.xml | 2 +-
tephra-examples/hbase-1.0/pom.xml | 2 +-
tephra-examples/hbase-1.1/pom.xml | 2 +-
tephra-examples/hbase-1.2/pom.xml | 2 +-
tephra-examples/pom.xml | 2 +-
tephra-hbase-compat-0.96/pom.xml | 2 +-
tephra-hbase-compat-0.98/pom.xml | 2 +-
tephra-hbase-compat-1.0-cdh/pom.xml | 2 +-
tephra-hbase-compat-1.0/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml | 2 +-
tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml | 2 +-
21 files changed, 22 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1649d74..e81dad4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,7 +28,7 @@
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Tephra</name>
<description>
@@ -165,7 +165,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-tephra.git</connection>
<url>https://git-wip-us.apache.org/repos/asf?p=incubator-tephra.git;a=summary</url>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-tephra.git</developerConnection>
- <tag>v0.11.0-incubating</tag>
+ <tag>HEAD</tag>
</scm>
<issueManagement>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-api/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-api/pom.xml b/tephra-api/pom.xml
index 0ed4e2e..9f386b2 100644
--- a/tephra-api/pom.xml
+++ b/tephra-api/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>tephra-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-core/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-core/pom.xml b/tephra-core/pom.xml
index 8f7104d..54c5440 100644
--- a/tephra-core/pom.xml
+++ b/tephra-core/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-distribution/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-distribution/pom.xml b/tephra-distribution/pom.xml
index 6df3d8f..e440ece 100644
--- a/tephra-distribution/pom.xml
+++ b/tephra-distribution/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>tephra-distribution</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-examples/cdh-5.7/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/cdh-5.7/pom.xml b/tephra-examples/cdh-5.7/pom.xml
index f83aad7..53efe12 100644
--- a/tephra-examples/cdh-5.7/pom.xml
+++ b/tephra-examples/cdh-5.7/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-examples</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-examples/cdh-5.8/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/cdh-5.8/pom.xml b/tephra-examples/cdh-5.8/pom.xml
index bb10753..f29476e 100644
--- a/tephra-examples/cdh-5.8/pom.xml
+++ b/tephra-examples/cdh-5.8/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-examples/hbase-0.96/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-0.96/pom.xml b/tephra-examples/hbase-0.96/pom.xml
index e481c8f..8c8d87d 100644
--- a/tephra-examples/hbase-0.96/pom.xml
+++ b/tephra-examples/hbase-0.96/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-examples/hbase-0.98/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-0.98/pom.xml b/tephra-examples/hbase-0.98/pom.xml
index 4e59a8e..f3de4f5 100644
--- a/tephra-examples/hbase-0.98/pom.xml
+++ b/tephra-examples/hbase-0.98/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-examples/hbase-1.0-cdh/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.0-cdh/pom.xml b/tephra-examples/hbase-1.0-cdh/pom.xml
index e8c7490..7382b87 100644
--- a/tephra-examples/hbase-1.0-cdh/pom.xml
+++ b/tephra-examples/hbase-1.0-cdh/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-examples/hbase-1.0/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.0/pom.xml b/tephra-examples/hbase-1.0/pom.xml
index 89597f2..52a8717 100644
--- a/tephra-examples/hbase-1.0/pom.xml
+++ b/tephra-examples/hbase-1.0/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>tephra-examples</artifactId>
<groupId>org.apache.tephra</groupId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-examples/hbase-1.1/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.1/pom.xml b/tephra-examples/hbase-1.1/pom.xml
index b377546..e15574c 100644
--- a/tephra-examples/hbase-1.1/pom.xml
+++ b/tephra-examples/hbase-1.1/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-examples</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-examples/hbase-1.2/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/hbase-1.2/pom.xml b/tephra-examples/hbase-1.2/pom.xml
index 4284a91..2cef197 100644
--- a/tephra-examples/hbase-1.2/pom.xml
+++ b/tephra-examples/hbase-1.2/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-examples</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-examples/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-examples/pom.xml b/tephra-examples/pom.xml
index 3d6ad4e..63d0a66 100644
--- a/tephra-examples/pom.xml
+++ b/tephra-examples/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-hbase-compat-0.96/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/pom.xml b/tephra-hbase-compat-0.96/pom.xml
index 282f385..0efc506 100644
--- a/tephra-hbase-compat-0.96/pom.xml
+++ b/tephra-hbase-compat-0.96/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-hbase-compat-0.98/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/pom.xml b/tephra-hbase-compat-0.98/pom.xml
index d278641..b83372b 100644
--- a/tephra-hbase-compat-0.98/pom.xml
+++ b/tephra-hbase-compat-0.98/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-hbase-compat-1.0-cdh/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/pom.xml b/tephra-hbase-compat-1.0-cdh/pom.xml
index 2b5d70d..31dbf56 100644
--- a/tephra-hbase-compat-1.0-cdh/pom.xml
+++ b/tephra-hbase-compat-1.0-cdh/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-hbase-compat-1.0/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/pom.xml b/tephra-hbase-compat-1.0/pom.xml
index c81b708..84e203d 100644
--- a/tephra-hbase-compat-1.0/pom.xml
+++ b/tephra-hbase-compat-1.0/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-hbase-compat-1.1-base/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/pom.xml b/tephra-hbase-compat-1.1-base/pom.xml
index 71a414d..1ae7829 100644
--- a/tephra-hbase-compat-1.1-base/pom.xml
+++ b/tephra-hbase-compat-1.1-base/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
index e6637c5..0ac15be 100644
--- a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
+++ b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.1-base</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
index f36762c..b152305 100644
--- a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
+++ b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2-cdh/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.1-base</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/14aca68a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
index 4f5981f..359af98 100644
--- a/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
+++ b/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.2/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.1-base</artifactId>
- <version>0.11.0-incubating</version>
+ <version>0.12.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>