You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2017/10/25 12:37:22 UTC
[01/23] hbase git commit: HBASE-19066 Correct the directory of
openjdk-8 for jenkins [Forced Update!]
Repository: hbase
Updated Branches:
refs/heads/HBASE-18410 d370fac77 -> 705b3fa98 (forced update)
HBASE-19066 Correct the directory of openjdk-8 for jenkins
Signed-off-by: Sean Busbey <bu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a31e8e34
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a31e8e34
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a31e8e34
Branch: refs/heads/HBASE-18410
Commit: a31e8e34ea2d6de9b40c251bfda8511dfc80f7f7
Parents: 93bac3d
Author: Chia-Ping Tsai <ch...@gmail.com>
Authored: Mon Oct 23 03:49:07 2017 +0800
Committer: Sean Busbey <bu...@apache.org>
Committed: Tue Oct 24 09:18:47 2017 -0500
----------------------------------------------------------------------
dev-support/Jenkinsfile | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a31e8e34/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 1f01a47..28efa61 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -121,7 +121,7 @@ curl -L -o personality.sh "${env.PROJET_PERSONALITY}"
TESTS = 'all,-unit,-findbugs'
// on branches that don't support jdk7, this will already be JAVA_HOME, so we'll end up not
// doing multijdk there.
- MULTIJDK = '/usr/lib/jvm/java-8-oracle'
+ MULTIJDK = '/usr/lib/jvm/java-8-openjdk-amd64'
OUTPUT_RELATIVE = "${env.OUTPUT_RELATIVE_GENERAL}"
OUTPUTDIR = "${env.OUTPUTDIR_GENERAL}"
}
@@ -211,7 +211,7 @@ curl -L -o personality.sh "${env.PROJET_PERSONALITY}"
OUTPUTDIR = "${env.OUTPUTDIR_JDK8}"
// This isn't strictly needed on branches that only support jdk8, but doesn't hurt
// and is needed on branches that do both jdk7 and jdk8
- SET_JAVA_HOME = '/usr/lib/jvm/java-8-oracle'
+ SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
}
steps {
unstash 'yetus'
[12/23] hbase git commit: HBASE-19077 Have
Region*CoprocessorEnvironment provide an ImmutableOnlineRegions
Posted by zh...@apache.org.
HBASE-19077 Have Region*CoprocessorEnvironment provide an ImmutableOnlineRegions
Change name of Interface OnlineRegions to MutableOnlineRegions.
Change name of Interface ImmutableOnlineRegions to OnlineRegions.
Did this since OnlineRegions is for consumer other than internals.
Add a getOnlineRegions to the RegionCoprocessorEnvironment and to
RegionServerCoprocessorEnvironment so CPs can 'access' local
Regions directly.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/95528d12
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/95528d12
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/95528d12
Branch: refs/heads/HBASE-18410
Commit: 95528d12f33eeccb1bd1e80b3441c4f14153dcd2
Parents: 2e64813
Author: Michael Stack <st...@apache.org>
Authored: Tue Oct 24 15:37:32 2017 -0700
Committer: Michael Stack <st...@apache.org>
Committed: Tue Oct 24 21:58:14 2017 -0700
----------------------------------------------------------------------
.../RegionCoprocessorEnvironment.java | 6 ++
.../RegionServerCoprocessorEnvironment.java | 6 ++
.../regionserver/ImmutableOnlineRegions.java | 60 --------------------
.../regionserver/MutableOnlineRegions.java | 44 ++++++++++++++
.../hbase/regionserver/OnlineRegions.java | 47 +++++++++------
.../regionserver/RegionCoprocessorHost.java | 6 ++
.../RegionServerCoprocessorHost.java | 7 +++
.../regionserver/RegionServerServices.java | 2 +-
.../regionserver/handler/OpenRegionHandler.java | 2 +-
.../security/token/TestTokenAuthentication.java | 6 ++
10 files changed, 108 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/95528d12/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java
index 8058a9d..05df33d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.metrics.MetricRegistry;
+import org.apache.hadoop.hbase.regionserver.OnlineRegions;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
@@ -40,6 +41,11 @@ public interface RegionCoprocessorEnvironment extends CoprocessorEnvironment<Reg
/** @return region information for the region this coprocessor is running on */
RegionInfo getRegionInfo();
+ /**
+ * @return Interface to Map of regions online on this RegionServer {@link #getServerName()}}.
+ */
+ OnlineRegions getOnlineRegions();
+
/** @return shared data between all instances of this coprocessor */
ConcurrentMap<String, Object> getSharedData();
http://git-wip-us.apache.org/repos/asf/hbase/blob/95528d12/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java
index d0a728e..116d86d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.metrics.MetricRegistry;
+import org.apache.hadoop.hbase.regionserver.OnlineRegions;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
@@ -36,6 +37,11 @@ public interface RegionServerCoprocessorEnvironment
ServerName getServerName();
/**
+ * @return Interface to Map of regions online on this RegionServer {@link #getServerName()}}.
+ */
+ OnlineRegions getOnlineRegions();
+
+ /**
* Be careful RPC'ing from a Coprocessor context.
* RPC's will fail, stall, retry, and/or crawl because the remote side is not online, is
* struggling or it is on the other side of a network partition. Any use of Connection from
http://git-wip-us.apache.org/repos/asf/hbase/blob/95528d12/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableOnlineRegions.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableOnlineRegions.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableOnlineRegions.java
deleted file mode 100644
index ffd1fa7..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableOnlineRegions.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.hbase.HBaseInterfaceAudience;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.yetus.audience.InterfaceStability;
-
-/**
- * Interface to Map of online regions. In the Map, the key is the region's
- * encoded name and the value is an {@link Region} instance.
- */
-@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
-@InterfaceStability.Evolving
-public interface ImmutableOnlineRegions {
-
- /**
- * Return {@link Region} instance.
- * Only works if caller is in same context, in same JVM. Region is not
- * serializable.
- * @param encodedRegionName
- * @return Region for the passed encoded <code>encodedRegionName</code> or
- * null if named region is not member of the online regions.
- */
- Region getRegion(String encodedRegionName);
-
- /**
- * Get all online regions of a table in this RS.
- * @param tableName
- * @return List of Region
- * @throws java.io.IOException
- */
- List<? extends Region> getRegions(TableName tableName) throws IOException;
-
- /**
- * Get all online regions in this RS.
- * @return List of online Region
- */
- List<? extends Region> getRegions();
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/95528d12/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableOnlineRegions.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableOnlineRegions.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableOnlineRegions.java
new file mode 100644
index 0000000..11c9af9
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableOnlineRegions.java
@@ -0,0 +1,44 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Interface to Map of online regions. In the Map, the key is the region's
+ * encoded name and the value is an {@link Region} instance.
+ */
+@InterfaceAudience.Private
+public interface MutableOnlineRegions extends OnlineRegions {
+
+ /**
+ * Add to online regions.
+ * @param r
+ */
+ void addRegion(final HRegion r);
+
+ /**
+ * Removes the given Region from the list of onlineRegions.
+ * @param r Region to remove.
+ * @param destination Destination, if any, null otherwise.
+ * @return True if we removed a region from online list.
+ */
+ boolean removeRegion(final HRegion r, ServerName destination);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/95528d12/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
index c9f1c47..25594b7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
@@ -18,28 +18,43 @@
*/
package org.apache.hadoop.hbase.regionserver;
-import org.apache.hadoop.hbase.ServerName;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.TableName;
import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
/**
- * Interface to Map of online regions. In the Map, the key is the region's
- * encoded name and the value is an {@link Region} instance.
+ * Provides read-only access to the Regions presently online on the
+ * current RegionServer
*/
-@InterfaceAudience.Private
-public interface OnlineRegions extends ImmutableOnlineRegions {
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
+@InterfaceStability.Evolving
+public interface OnlineRegions {
/**
- * Add to online regions.
- * @param r
+ * Return {@link Region} instance.
+ * Only works if caller is in same context, in same JVM. Region is not
+ * serializable.
+ * @param encodedRegionName
+ * @return Region for the passed encoded <code>encodedRegionName</code> or
+ * null if named region is not member of the online regions.
*/
- void addRegion(final HRegion r);
+ Region getRegion(String encodedRegionName);
- /**
- * This method removes Region corresponding to hri from the Map of onlineRegions.
- *
- * @param r Region to remove.
- * @param destination Destination, if any, null otherwise.
- * @return True if we removed a region from online list.
- */
- boolean removeRegion(final HRegion r, ServerName destination);
+ /**
+ * Get all online regions of a table in this RS.
+ * @param tableName
+ * @return List of Region
+ * @throws java.io.IOException
+ */
+ List<? extends Region> getRegions(TableName tableName) throws IOException;
+
+ /**
+ * Get all online regions in this RS.
+ * @return List of online Region
+ */
+ List<? extends Region> getRegions();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/95528d12/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index 735d7ba..42d7ac9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -117,6 +117,7 @@ public class RegionCoprocessorHost
private final MetricRegistry metricRegistry;
private final Connection connection;
private final ServerName serverName;
+ private final OnlineRegions onlineRegions;
/**
* Constructor
@@ -132,6 +133,7 @@ public class RegionCoprocessorHost
this.connection = services != null? services.getConnection(): null;
this.serverName = services != null? services.getServerName(): null;
this.sharedData = sharedData;
+ this.onlineRegions = services;
this.metricRegistry =
MetricsCoprocessor.createRegistryForRegionCoprocessor(impl.getClass().getName());
}
@@ -142,6 +144,10 @@ public class RegionCoprocessorHost
return region;
}
+ public OnlineRegions getOnlineRegions() {
+ return this.onlineRegions;
+ }
+
@Override
public Connection getConnection() {
return this.connection;
http://git-wip-us.apache.org/repos/asf/hbase/blob/95528d12/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
index 33be479..27a3e20 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
@@ -206,6 +206,7 @@ public class RegionServerCoprocessorHost extends
private final MetricRegistry metricRegistry;
private final Connection connection;
private final ServerName serverName;
+ private final OnlineRegions onlineRegions;
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="BC_UNCONFIRMED_CAST",
justification="Intentional; FB has trouble detecting isAssignableFrom")
@@ -216,6 +217,7 @@ public class RegionServerCoprocessorHost extends
for (Service service : impl.getServices()) {
services.registerService(service);
}
+ this.onlineRegions = services;
this.connection = services.getConnection();
this.serverName = services.getServerName();
this.metricRegistry =
@@ -223,6 +225,11 @@ public class RegionServerCoprocessorHost extends
}
@Override
+ public OnlineRegions getOnlineRegions() {
+ return this.onlineRegions;
+ }
+
+ @Override
public ServerName getServerName() {
return this.serverName;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/95528d12/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
index af883a3..2334302 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
@@ -48,7 +48,7 @@ import com.google.protobuf.Service;
* the code base.
*/
@InterfaceAudience.Private
-public interface RegionServerServices extends Server, OnlineRegions, FavoredNodesForRegion {
+public interface RegionServerServices extends Server, MutableOnlineRegions, FavoredNodesForRegion {
/** @return the WAL for a particular region. Pass null for getting the
* default (common) WAL */
http://git-wip-us.apache.org/repos/asf/hbase/blob/95528d12/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
index e664cd5..e4b3ed2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
@@ -121,7 +121,7 @@ public class OpenRegionHandler extends EventHandler {
return;
}
- // Successful region open, and add it to OnlineRegions
+ // Successful region open, and add it to MutableOnlineRegions
this.rsServices.addRegion(region);
openSuccessful = true;
http://git-wip-us.apache.org/repos/asf/hbase/blob/95528d12/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java
index 11af0de..e849b30 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.ipc.SimpleRpcServer;
import org.apache.hadoop.hbase.metrics.MetricRegistry;
import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.OnlineRegions;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.security.SecurityInfo;
import org.apache.hadoop.hbase.security.User;
@@ -273,6 +274,11 @@ public class TestTokenAuthentication {
public HRegion getRegion() { return null; }
@Override
+ public OnlineRegions getOnlineRegions() {
+ return null;
+ }
+
+ @Override
public void startup() throws IOException {}
@Override
[23/23] hbase git commit: HBASE-17678 FilterList with MUST_PASS_ONE
may lead to redundant cells returned
Posted by zh...@apache.org.
HBASE-17678 FilterList with MUST_PASS_ONE may lead to redundant cells returned
Signed-off-by: tedyu <yu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/45910981
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/45910981
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/45910981
Branch: refs/heads/HBASE-18410
Commit: 45910981b78ea2d238b9bcd2caef1270ca0261b4
Parents: 4e1c720
Author: huzheng <op...@gmail.com>
Authored: Sat May 27 16:58:00 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Wed Oct 25 20:36:19 2017 +0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/filter/FilterList.java | 74 +++++++++-
.../hadoop/hbase/filter/TestFilterList.java | 136 +++++++++++++++++--
2 files changed, 200 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/45910981/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index 2f11472..3493082 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -67,6 +67,14 @@ final public class FilterList extends FilterBase {
private final List<Filter> filters;
private Collection<Filter> seekHintFilters = new ArrayList<Filter>();
+ /**
+ * Save previous return code and previous cell for every filter in filter list. For MUST_PASS_ONE,
+ * we use the previous return code to decide whether we should pass current cell encountered to
+ * the filter. For MUST_PASS_ALL, the two list are meaningless.
+ */
+ private List<ReturnCode> prevFilterRCList = null;
+ private List<Cell> prevCellList = null;
+
/** Reference Cell used by {@link #transformCell(Cell)} for validation purpose. */
private Cell referenceCell = null;
@@ -88,6 +96,7 @@ final public class FilterList extends FilterBase {
public FilterList(final List<Filter> rowFilters) {
reversed = getReversed(rowFilters, reversed);
this.filters = new ArrayList<>(rowFilters);
+ initPrevListForMustPassOne(rowFilters.size());
}
/**
@@ -107,6 +116,7 @@ final public class FilterList extends FilterBase {
public FilterList(final Operator operator) {
this.operator = operator;
this.filters = new ArrayList<>();
+ initPrevListForMustPassOne(filters.size());
}
/**
@@ -118,6 +128,7 @@ final public class FilterList extends FilterBase {
public FilterList(final Operator operator, final List<Filter> rowFilters) {
this(rowFilters);
this.operator = operator;
+ initPrevListForMustPassOne(rowFilters.size());
}
/**
@@ -129,8 +140,21 @@ final public class FilterList extends FilterBase {
public FilterList(final Operator operator, final Filter... rowFilters) {
this(rowFilters);
this.operator = operator;
+ initPrevListForMustPassOne(rowFilters.length);
+ }
+
+ public void initPrevListForMustPassOne(int size) {
+ if (operator == Operator.MUST_PASS_ONE) {
+ if (this.prevCellList == null) {
+ prevFilterRCList = new ArrayList<>(Collections.nCopies(size, null));
+ }
+ if (this.prevCellList == null) {
+ prevCellList = new ArrayList<>(Collections.nCopies(size, null));
+ }
+ }
}
+
/**
* Get the operator.
*
@@ -185,6 +209,10 @@ final public class FilterList extends FilterBase {
public void addFilter(List<Filter> filters) {
checkReversed(filters, isReversed());
this.filters.addAll(filters);
+ if (operator == Operator.MUST_PASS_ONE) {
+ this.prevFilterRCList.addAll(Collections.nCopies(filters.size(), null));
+ this.prevCellList.addAll(Collections.nCopies(filters.size(), null));
+ }
}
/**
@@ -201,6 +229,10 @@ final public class FilterList extends FilterBase {
int listize = filters.size();
for (int i = 0; i < listize; i++) {
filters.get(i).reset();
+ if (operator == Operator.MUST_PASS_ONE) {
+ prevFilterRCList.set(i, null);
+ prevCellList.set(i, null);
+ }
}
seekHintFilters.clear();
}
@@ -283,6 +315,41 @@ final public class FilterList extends FilterBase {
return this.transformedCell;
}
+ /**
+ * For MUST_PASS_ONE, we cannot make sure that when filter-A in filter list return NEXT_COL then
+ * the next cell passing to filterList will be the first cell in next column, because if filter-B
+ * in filter list return SKIP, then the filter list will return SKIP. In this case, we should pass
+ * the cell following the previous cell, and it's possible that the next cell has the same column
+ * as the previous cell even if filter-A has NEXT_COL returned for the previous cell. So we should
+ * save the previous cell and the return code list when checking previous cell for every filter in
+ * filter list, and verify if currentCell fit the previous return code, if fit then pass the currentCell
+ * to the corresponding filter. (HBASE-17678)
+ */
+ private boolean shouldPassCurrentCellToFilter(Cell prevCell, Cell currentCell, int filterIdx)
+ throws IOException {
+ ReturnCode prevCode = this.prevFilterRCList.get(filterIdx);
+ if (prevCell == null || prevCode == null) {
+ return true;
+ }
+ switch (prevCode) {
+ case INCLUDE:
+ case SKIP:
+ return true;
+ case SEEK_NEXT_USING_HINT:
+ Cell nextHintCell = getNextCellHint(prevCell);
+ return nextHintCell == null
+ || CellComparatorImpl.COMPARATOR.compare(currentCell, nextHintCell) >= 0;
+ case NEXT_COL:
+ case INCLUDE_AND_NEXT_COL:
+ return !CellUtil.matchingRowColumn(prevCell, currentCell);
+ case NEXT_ROW:
+ case INCLUDE_AND_SEEK_NEXT_ROW:
+ return !CellUtil.matchingRows(prevCell, currentCell);
+ default:
+ throw new IllegalStateException("Received code is not valid.");
+ }
+ }
+
@Override
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="SF_SWITCH_FALLTHROUGH",
justification="Intentional")
@@ -331,12 +398,17 @@ final public class FilterList extends FilterBase {
}
}
} else if (operator == Operator.MUST_PASS_ONE) {
- if (filter.filterAllRemaining()) {
+ Cell prevCell = this.prevCellList.get(i);
+ if (filter.filterAllRemaining() || !shouldPassCurrentCellToFilter(prevCell, c, i)) {
seenNonHintReturnCode = true;
continue;
}
ReturnCode localRC = filter.filterKeyValue(c);
+ // Update previous cell and return code we encountered.
+ prevFilterRCList.set(i, localRC);
+ prevCellList.set(i, c);
+
if (localRC != ReturnCode.SEEK_NEXT_USING_HINT) {
seenNonHintReturnCode = true;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/45910981/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index 9ad0962..46d44de 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -18,32 +18,34 @@
*/
package org.apache.hadoop.hbase.filter;
-import org.apache.hadoop.hbase.CompareOperator;
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparatorImpl;
+import org.apache.hadoop.hbase.CompareOperator;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.testclassification.FilterTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+
/**
* Tests filter sets
*
@@ -593,5 +595,121 @@ public class TestFilterList {
assertEquals(Filter.ReturnCode.SKIP, flist.filterKeyValue(kvQual3));
}
+ @Test
+ public void testWithMultiVersionsInSameRow() throws Exception {
+ FilterList filterList01 =
+ new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, 0));
+
+ KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
+ 1, Bytes.toBytes("value"));
+ KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
+ 2, Bytes.toBytes("value"));
+ KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
+ 3, Bytes.toBytes("value"));
+
+ assertEquals(filterList01.filterKeyValue(kv1), ReturnCode.INCLUDE_AND_NEXT_COL);
+ assertEquals(filterList01.filterKeyValue(kv2), ReturnCode.SKIP);
+ assertEquals(filterList01.filterKeyValue(kv3), ReturnCode.SKIP);
+
+ FilterList filterList11 =
+ new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, 1));
+
+ assertEquals(filterList11.filterKeyValue(kv1), ReturnCode.SKIP);
+ assertEquals(filterList11.filterKeyValue(kv2), ReturnCode.SKIP);
+ assertEquals(filterList11.filterKeyValue(kv3), ReturnCode.SKIP);
+ }
+
+ @Test
+ public void testMPONEWithSeekNextUsingHint() throws Exception {
+ byte[] col = Bytes.toBytes("c");
+ FilterList filterList =
+ new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, col));
+
+ KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
+ Bytes.toBytes("value"));
+ KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("b"), 2,
+ Bytes.toBytes("value"));
+ KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 3,
+ Bytes.toBytes("value"));
+ KeyValue kv4 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 4,
+ Bytes.toBytes("value"));
+
+ assertEquals(filterList.filterKeyValue(kv1), ReturnCode.SEEK_NEXT_USING_HINT);
+ assertEquals(filterList.filterKeyValue(kv2), ReturnCode.SKIP);
+ assertEquals(filterList.filterKeyValue(kv3), ReturnCode.INCLUDE_AND_NEXT_COL);
+ assertEquals(filterList.filterKeyValue(kv4), ReturnCode.SKIP);
+ }
+
+ private static class MockFilter extends FilterBase {
+ private ReturnCode targetRetCode;
+ public boolean didCellPassToTheFilter = false;
+
+ public MockFilter(ReturnCode targetRetCode) {
+ this.targetRetCode = targetRetCode;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell v) throws IOException {
+ this.didCellPassToTheFilter = true;
+ return targetRetCode;
+ }
+ }
+
+ @Test
+ public void testShouldPassCurrentCellToFilter() throws IOException {
+ KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
+ Bytes.toBytes("value"));
+ KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 2,
+ Bytes.toBytes("value"));
+ KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("b"), 3,
+ Bytes.toBytes("value"));
+ KeyValue kv4 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 4,
+ Bytes.toBytes("value"));
+
+ MockFilter mockFilter = new MockFilter(ReturnCode.NEXT_COL);
+ FilterList filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter);
+
+ filter.filterKeyValue(kv1);
+ assertTrue(mockFilter.didCellPassToTheFilter);
+
+ mockFilter.didCellPassToTheFilter = false;
+ filter.filterKeyValue(kv2);
+ assertFalse(mockFilter.didCellPassToTheFilter);
+
+ mockFilter.didCellPassToTheFilter = false;
+ filter.filterKeyValue(kv3);
+ assertTrue(mockFilter.didCellPassToTheFilter);
+
+ mockFilter = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
+ filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter);
+
+ filter.filterKeyValue(kv1);
+ assertTrue(mockFilter.didCellPassToTheFilter);
+
+ mockFilter.didCellPassToTheFilter = false;
+ filter.filterKeyValue(kv2);
+ assertFalse(mockFilter.didCellPassToTheFilter);
+
+ mockFilter.didCellPassToTheFilter = false;
+ filter.filterKeyValue(kv3);
+ assertTrue(mockFilter.didCellPassToTheFilter);
+
+ mockFilter = new MockFilter(ReturnCode.NEXT_ROW);
+ filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter);
+ filter.filterKeyValue(kv1);
+ assertTrue(mockFilter.didCellPassToTheFilter);
+
+ mockFilter.didCellPassToTheFilter = false;
+ filter.filterKeyValue(kv2);
+ assertFalse(mockFilter.didCellPassToTheFilter);
+
+ mockFilter.didCellPassToTheFilter = false;
+ filter.filterKeyValue(kv3);
+ assertFalse(mockFilter.didCellPassToTheFilter);
+
+ mockFilter.didCellPassToTheFilter = false;
+ filter.filterKeyValue(kv4);
+ assertTrue(mockFilter.didCellPassToTheFilter);
+ }
}
[20/23] hbase git commit: HBASE-19057 Fix other code review comments
about FilterList improvement
Posted by zh...@apache.org.
HBASE-19057 Fix other code review comments about FilterList improvement
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/705b3fa9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/705b3fa9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/705b3fa9
Branch: refs/heads/HBASE-18410
Commit: 705b3fa98c97806c7eba63617a99f62d829400d1
Parents: fcaf71d
Author: huzheng <op...@gmail.com>
Authored: Tue Oct 24 15:30:55 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Wed Oct 25 20:36:19 2017 +0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/filter/FilterList.java | 24 ++++++++++-----
.../hadoop/hbase/filter/FilterListBase.java | 16 ++++++++--
.../hadoop/hbase/filter/FilterListWithAND.java | 12 ++++----
.../hadoop/hbase/filter/FilterListWithOR.java | 31 +++++++++++---------
.../hadoop/hbase/filter/TestFilterList.java | 8 +++--
5 files changed, 59 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/705b3fa9/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index e87f1b3..d4242ae 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -21,17 +21,12 @@ package org.apache.hadoop.hbase.filter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Collections;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparatorImpl;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
@@ -170,8 +165,23 @@ final public class FilterList extends FilterBase {
return filterListBase.transformCell(c);
}
- ReturnCode internalFilterKeyValue(Cell c, Cell currentTransformedCell) throws IOException {
- return this.filterListBase.internalFilterKeyValue(c, currentTransformedCell);
+ /**
+ * Internal implementation of {@link #filterKeyValue(Cell)}. Compared to the
+ * {@link #filterKeyValue(Cell)} method, this method accepts an additional parameter named
+ * transformedCell. This parameter indicates the initial value of transformed cell before this
+ * filter operation. <br/>
+ * For FilterList, we can consider a filter list as a node in a tree. sub-filters of the filter
+ * list are children of the relative node. The logic of transforming cell of a filter list, well,
+ * we can consider it as the process of post-order tree traverse. For a node , Before we traverse
+ * the current child, we should set the traverse result (transformed cell) of previous node(s) as
+ * the initial value. so the additional currentTransformedCell parameter is needed (HBASE-18879).
+ * @param c The cell in question.
+ * @param transformedCell The transformed cell of previous filter(s)
+ * @return ReturnCode of this filter operation.
+ * @throws IOException
+ */
+ ReturnCode internalFilterKeyValue(Cell c, Cell transformedCell) throws IOException {
+ return this.filterListBase.internalFilterKeyValue(c, transformedCell);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/705b3fa9/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java
index 60b0dc1..f92d2e7 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java
@@ -107,8 +107,20 @@ public abstract class FilterListBase extends FilterBase {
return cell;
}
- abstract ReturnCode internalFilterKeyValue(Cell c, Cell currentTransformedCell)
- throws IOException;
+ /**
+ * Internal implementation of {@link #filterKeyValue(Cell)}
+ * @param c The cell in question.
+ * @param transformedCell The transformed cell of previous filter(s)
+ * @return ReturnCode of this filter operation.
+ * @throws IOException
+ * @see org.apache.hadoop.hbase.filter.FilterList#internalFilterKeyValue(Cell, Cell)
+ */
+ abstract ReturnCode internalFilterKeyValue(Cell c, Cell transformedCell) throws IOException;
+
+ @Override
+ public ReturnCode filterKeyValue(Cell c) throws IOException {
+ return internalFilterKeyValue(c, c);
+ }
/**
* Filters that never filter by modifying the returned List of Cells can inherit this
http://git-wip-us.apache.org/repos/asf/hbase/blob/705b3fa9/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java
index 4909dfd..9217ff9 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java
@@ -148,12 +148,12 @@ public class FilterListWithAND extends FilterListBase {
}
@Override
- ReturnCode internalFilterKeyValue(Cell c, Cell currentTransformedCell) throws IOException {
+ ReturnCode internalFilterKeyValue(Cell c, Cell transformedCell) throws IOException {
if (isEmpty()) {
return ReturnCode.INCLUDE;
}
ReturnCode rc = ReturnCode.INCLUDE;
- Cell transformed = currentTransformedCell;
+ Cell transformed = transformedCell;
this.referenceCell = c;
this.seekHintFilter.clear();
for (int i = 0, n = filters.size(); i < n; i++) {
@@ -186,11 +186,6 @@ public class FilterListWithAND extends FilterListBase {
}
@Override
- public ReturnCode filterKeyValue(Cell c) throws IOException {
- return internalFilterKeyValue(c, c);
- }
-
- @Override
public void reset() throws IOException {
for (int i = 0, n = filters.size(); i < n; i++) {
filters.get(i).reset();
@@ -222,6 +217,9 @@ public class FilterListWithAND extends FilterListBase {
for (int i = 0, n = filters.size(); i < n; i++) {
Filter filter = filters.get(i);
if (filter.filterAllRemaining() || filter.filterRowKey(firstRowCell)) {
+ // Can't just return true here, because there are some filters (such as PrefixFilter) which
+ // will catch the row changed event by filterRowKey(). If we return early here, those
+ // filters will have no chance to update their row state.
retVal = true;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/705b3fa9/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java
index 31e2a55..e51915b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java
@@ -81,9 +81,8 @@ public class FilterListWithOR extends FilterListBase {
* to the filter, if row mismatch or row match but column family mismatch. (HBASE-18368)
* @see org.apache.hadoop.hbase.filter.Filter.ReturnCode
*/
- private boolean shouldPassCurrentCellToFilter(Cell prevCell, Cell currentCell, int filterIdx)
- throws IOException {
- ReturnCode prevCode = this.prevFilterRCList.get(filterIdx);
+ private boolean shouldPassCurrentCellToFilter(Cell prevCell, Cell currentCell,
+ ReturnCode prevCode) throws IOException {
if (prevCell == null || prevCode == null) {
return true;
}
@@ -96,11 +95,13 @@ public class FilterListWithOR extends FilterListBase {
return nextHintCell == null || this.compareCell(currentCell, nextHintCell) >= 0;
case NEXT_COL:
case INCLUDE_AND_NEXT_COL:
- return !CellUtil.matchingRowColumn(prevCell, currentCell);
+ // Once row changed, reset() will clear prevCells, so we need not to compare their rows
+ // because rows are the same here.
+ return !CellUtil.matchingColumn(prevCell, currentCell);
case NEXT_ROW:
case INCLUDE_AND_SEEK_NEXT_ROW:
- return !CellUtil.matchingRows(prevCell, currentCell)
- || !CellUtil.matchingFamily(prevCell, currentCell);
+ // As described above, rows are definitely the same, so we only compare the family.
+ return !CellUtil.matchingFamily(prevCell, currentCell);
default:
throw new IllegalStateException("Received code is not valid.");
}
@@ -181,6 +182,9 @@ public class FilterListWithOR extends FilterListBase {
if (isInReturnCodes(rc, ReturnCode.INCLUDE)) {
return ReturnCode.INCLUDE;
}
+ if (isInReturnCodes(rc, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.NEXT_COL;
+ }
if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL,
ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
return ReturnCode.INCLUDE_AND_NEXT_COL;
@@ -242,19 +246,20 @@ public class FilterListWithOR extends FilterListBase {
}
@Override
- ReturnCode internalFilterKeyValue(Cell c, Cell currentTransformCell) throws IOException {
+ ReturnCode internalFilterKeyValue(Cell c, Cell transformCell) throws IOException {
if (isEmpty()) {
return ReturnCode.INCLUDE;
}
ReturnCode rc = null;
boolean everyFilterReturnHint = true;
- Cell transformed = currentTransformCell;
+ Cell transformed = transformCell;
this.referenceCell = c;
for (int i = 0, n = filters.size(); i < n; i++) {
Filter filter = filters.get(i);
Cell prevCell = this.prevCellList.get(i);
- if (filter.filterAllRemaining() || !shouldPassCurrentCellToFilter(prevCell, c, i)) {
+ ReturnCode prevCode = this.prevFilterRCList.get(i);
+ if (filter.filterAllRemaining() || !shouldPassCurrentCellToFilter(prevCell, c, prevCode)) {
everyFilterReturnHint = false;
continue;
}
@@ -295,11 +300,6 @@ public class FilterListWithOR extends FilterListBase {
}
@Override
- public ReturnCode filterKeyValue(Cell c) throws IOException {
- return internalFilterKeyValue(c, c);
- }
-
- @Override
public void reset() throws IOException {
for (int i = 0, n = filters.size(); i < n; i++) {
filters.get(i).reset();
@@ -332,6 +332,9 @@ public class FilterListWithOR extends FilterListBase {
for (int i = 0, n = filters.size(); i < n; i++) {
Filter filter = filters.get(i);
if (!filter.filterAllRemaining() && !filter.filterRowKey(firstRowCell)) {
+ // Can't just return false here, because there are some filters (such as PrefixFilter) which
+ // will catch the row changed event by filterRowKey(). If we return early here, those
+ // filters will have no chance to update their row state.
retVal = false;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/705b3fa9/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index d420b02..8c83cf6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -330,7 +330,7 @@ public class TestFilterList {
* @throws Exception
*/
@Test
- public void testFilterListWithInclusiveStopFilteMustPassOne() throws Exception {
+ public void testFilterListWithInclusiveStopFilterMustPassOne() throws Exception {
byte[] r1 = Bytes.toBytes("Row1");
byte[] r11 = Bytes.toBytes("Row11");
byte[] r2 = Bytes.toBytes("Row2");
@@ -351,11 +351,13 @@ public class TestFilterList {
public AlwaysNextColFilter() {
super();
}
+
@Override
public ReturnCode filterKeyValue(Cell v) {
return ReturnCode.NEXT_COL;
}
- public static AlwaysNextColFilter parseFrom(final byte [] pbBytes)
+
+ public static AlwaysNextColFilter parseFrom(final byte[] pbBytes)
throws DeserializationException {
return new AlwaysNextColFilter();
}
@@ -701,6 +703,7 @@ public class TestFilterList {
filter.filterKeyValue(kv3);
assertFalse(mockFilter.didCellPassToTheFilter);
+ filter.reset();
mockFilter.didCellPassToTheFilter = false;
filter.filterKeyValue(kv4);
assertTrue(mockFilter.didCellPassToTheFilter);
@@ -718,6 +721,7 @@ public class TestFilterList {
filter.filterKeyValue(kv3);
assertFalse(mockFilter.didCellPassToTheFilter);
+ filter.reset();
mockFilter.didCellPassToTheFilter = false;
filter.filterKeyValue(kv4);
assertTrue(mockFilter.didCellPassToTheFilter);
[08/23] hbase git commit: HBASE-19054 switch precommit image to one
from maven
Posted by zh...@apache.org.
HBASE-19054 switch precommit image to one from maven
Amending Author: Mike Drob <md...@apache.org>
Signed-off-by: Sean Busbey <bu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eee3b018
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eee3b018
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eee3b018
Branch: refs/heads/HBASE-18410
Commit: eee3b0180ead73c09b33f9583bfee9c01bc3aed2
Parents: 9353c59
Author: Misty Stanley-Jones <mi...@apache.org>
Authored: Tue Oct 24 14:50:23 2017 -0500
Committer: Mike Drob <md...@apache.org>
Committed: Tue Oct 24 14:51:16 2017 -0500
----------------------------------------------------------------------
dev-support/docker/Dockerfile | 143 ++++---------------------------------
1 file changed, 15 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/eee3b018/dev-support/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index da5f32e..49ad14d 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -1,4 +1,3 @@
-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -18,142 +17,30 @@
# Dockerfile for installing the necessary dependencies for building Hadoop.
# See BUILDING.txt.
+FROM maven:3.5-jdk-8
-FROM ubuntu:trusty
-
-WORKDIR /root
-
-ENV DEBIAN_FRONTEND noninteractive
-ENV DEBCONF_TERSE true
-
-######
-# Install common dependencies from packages
-#
-# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
-# Ubuntu Java. See Java section below!
-######
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
- build-essential \
- bzip2 \
- cmake \
- curl \
- doxygen \
- fuse \
- g++ \
- gcc \
- git \
- gnupg-agent \
- make \
- libbz2-dev \
- libcurl4-openssl-dev \
- libfuse-dev \
- libperl-critic-perl \
- libprotobuf-dev \
- libprotoc-dev \
- libsnappy-dev \
- libssl-dev \
- libtool \
- pinentry-curses \
- pkg-config \
- protobuf-compiler \
- protobuf-c-compiler \
- python \
- python2.7 \
- python-pip \
- rsync \
- snappy \
- zlib1g-dev \
- wget
-
-#######
-# OpenJDK 8
-#######
-
-RUN echo "dot_style = mega" > "/root/.wgetrc"
-RUN echo "quiet = on" >> "/root/.wgetrc"
-
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y software-properties-common
-RUN add-apt-repository -y ppa:openjdk-r/ppa
-RUN apt-get -q update
-RUN apt-get -q install --no-install-recommends -y openjdk-8-jdk
-RUN update-alternatives --config java
-RUN update-alternatives --config javac
-
-####
-# Apps that require Java
-###
RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
- ant \
- maven
-
-# Fixing the Apache commons / Maven dependency problem under Ubuntu:
-# See http://wiki.apache.org/commons/VfsProblems
-RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang3-3.6.jar .
+ bats \
+ findbugs \
+ libperl-critic-perl \
+ pylint \
+ python-dateutil \
+ rsync \
+ ruby \
+ shellcheck \
+ wget \
+ && \
+ gem install rake rubocop ruby-lint
-######
-# Install findbugs
-######
-RUN mkdir -p /opt/findbugs && \
- curl -L -s -S \
- https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \
- -o /opt/findbugs.tar.gz && \
- tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs
-ENV FINDBUGS_HOME /opt/findbugs
-
-####
-# Install shellcheck
-####
-RUN apt-get -q install -y cabal-install
-RUN mkdir /root/.cabal
-RUN echo "remote-repo: hackage.fpcomplete.com:http://hackage.fpcomplete.com/" >> /root/.cabal/config
-#RUN echo "remote-repo: hackage.haskell.org:http://hackage.haskell.org/" > /root/.cabal/config
-RUN echo "remote-repo-cache: /root/.cabal/packages" >> /root/.cabal/config
-RUN cabal update
-RUN cabal install shellcheck --global
-
-####
-# Install bats
-####
-RUN add-apt-repository -y ppa:duggan/bats
-RUN apt-get -q update
-RUN apt-get -q install --no-install-recommends -y bats
-
-####
-# Install pylint
-####
-RUN pip install pylint
-
-####
-# Install dateutil.parser
-####
-RUN pip install python-dateutil
-
-####
-# Install Ruby 2, based on Yetus 0.4.0 dockerfile
-###
-RUN echo 'gem: --no-rdoc --no-ri' >> /root/.gemrc
-RUN apt-add-repository ppa:brightbox/ruby-ng
-RUN apt-get -q update
-
-RUN apt-get -q install --no-install-recommends -y ruby2.2 ruby-switch
-RUN ruby-switch --set ruby2.2
-
-####
-# Install rubocop
-###
-RUN gem install rake
-RUN gem install rubocop
-
-####
-# Install ruby-lint
-###
-RUN gem install ruby-lint
+ENV FINDBUGS_HOME /usr
###
# Avoid out of memory errors in builds
###
ENV MAVEN_OPTS -Xmx3g
+CMD /bin/bash
+
###
# Everything past this point is either not needed for testing or breaks Yetus.
# So tell Yetus not to read the rest of the file:
[11/23] hbase git commit: HBASE-18754 (addendum) close the input
resource
Posted by zh...@apache.org.
HBASE-18754 (addendum) close the input resource
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2e648138
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2e648138
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2e648138
Branch: refs/heads/HBASE-18410
Commit: 2e6481386b80c2306e0e12a9c7cd137b63eaa006
Parents: dd70cc3
Author: Chia-Ping Tsai <ch...@gmail.com>
Authored: Wed Oct 25 12:31:37 2017 +0800
Committer: Chia-Ping Tsai <ch...@gmail.com>
Committed: Wed Oct 25 12:31:37 2017 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e648138/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
index 08d9853..203bea3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
@@ -199,8 +199,9 @@ public abstract class TimeRangeTracker {
ProtobufUtil.mergeFrom(builder, data, pblen, data.length - pblen);
return TimeRangeTracker.create(type, builder.getFrom(), builder.getTo());
} else {
- DataInputStream in = new DataInputStream(new ByteArrayInputStream(data));
- return TimeRangeTracker.create(type, in.readLong(), in.readLong());
+ try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(data))) {
+ return TimeRangeTracker.create(type, in.readLong(), in.readLong());
+ }
}
}
[04/23] hbase git commit: HBASE-19074 Miscellaneous Observer cleanups
Breaks MemStoreSize into MemStoreSize (read-only) and MemStoreSizing
(read/write). MemStoreSize we allow to Coprocesors. MemStoreSizing we use
internally doing MemStore accounting.
Posted by zh...@apache.org.
HBASE-19074 Miscellaneous Observer cleanups
Breaks MemStoreSize into MemStoreSize (read-only) and MemStoreSizing
(read/write). MemStoreSize we allow to Coprocesors. MemStoreSizing we
use internally doing MemStore accounting.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cb506fd4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cb506fd4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cb506fd4
Branch: refs/heads/HBASE-18410
Commit: cb506fd4019556ec5535310b38e99b1ebfdf80bb
Parents: 9716f62
Author: Michael Stack <st...@apache.org>
Authored: Mon Oct 23 20:57:46 2017 -0700
Committer: Michael Stack <st...@apache.org>
Committed: Tue Oct 24 10:02:24 2017 -0700
----------------------------------------------------------------------
.../hbase/coprocessor/RegionObserver.java | 6 ++
.../hadoop/hbase/coprocessor/WALObserver.java | 6 ++
.../hbase/regionserver/AbstractMemStore.java | 28 ++---
.../regionserver/CellArrayImmutableSegment.java | 5 +-
.../regionserver/CellChunkImmutableSegment.java | 6 +-
.../hbase/regionserver/CompactingMemStore.java | 20 ++--
.../hbase/regionserver/CompactionPipeline.java | 22 ++--
.../regionserver/CompositeImmutableSegment.java | 4 +-
.../hbase/regionserver/DefaultMemStore.java | 16 +--
.../hadoop/hbase/regionserver/HRegion.java | 61 ++++++-----
.../hadoop/hbase/regionserver/HStore.java | 18 ++--
.../hadoop/hbase/regionserver/MemStore.java | 12 +--
.../hadoop/hbase/regionserver/MemStoreSize.java | 68 ++++--------
.../hbase/regionserver/MemStoreSizing.java | 103 +++++++++++++++++++
.../MiniBatchOperationInProgress.java | 3 +-
.../hbase/regionserver/MutableSegment.java | 16 ++-
.../regionserver/RegionServerAccounting.java | 9 +-
.../hadoop/hbase/regionserver/Segment.java | 16 +--
.../hbase/regionserver/SegmentFactory.java | 7 +-
.../TestCompactingToCellFlatMapMemStore.java | 8 +-
.../hbase/regionserver/TestDefaultMemStore.java | 4 +-
.../hadoop/hbase/regionserver/TestHStore.java | 86 ++++++++--------
.../regionserver/wal/AbstractTestWALReplay.java | 8 +-
23 files changed, 309 insertions(+), 223 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
index 815daf1..5c89149 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
@@ -880,7 +880,10 @@ public interface RegionObserver {
* Called before a {@link WALEdit}
* replayed for this region.
* @param ctx the environment provided by the region server
+ * @deprecated Since hbase-2.0.0. No replacement. To be removed in hbase-3.0.0 and replaced
+ * with something that doesn't expose IntefaceAudience.Private classes.
*/
+ @Deprecated
default void preWALRestore(ObserverContext<? extends RegionCoprocessorEnvironment> ctx,
RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {}
@@ -888,7 +891,10 @@ public interface RegionObserver {
* Called after a {@link WALEdit}
* replayed for this region.
* @param ctx the environment provided by the region server
+ * @deprecated Since hbase-2.0.0. No replacement. To be removed in hbase-3.0.0 and replaced
+ * with something that doesn't expose IntefaceAudience.Private classes.
*/
+ @Deprecated
default void postWALRestore(ObserverContext<? extends RegionCoprocessorEnvironment> ctx,
RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java
index 2190abf..ec8518b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java
@@ -72,8 +72,11 @@ public interface WALObserver {
* is writen to WAL.
*
* @return true if default behavior should be bypassed, false otherwise
+ * @deprecated Since hbase-2.0.0. To be replaced with an alternative that does not expose
+ * InterfaceAudience classes such as WALKey and WALEdit. Will be removed in hbase-3.0.0.
*/
// TODO: return value is not used
+ @Deprecated
default boolean preWALWrite(ObserverContext<? extends WALCoprocessorEnvironment> ctx,
RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
return false;
@@ -82,7 +85,10 @@ public interface WALObserver {
/**
* Called after a {@link WALEdit}
* is writen to WAL.
+ * @deprecated Since hbase-2.0.0. To be replaced with an alternative that does not expose
+ * InterfaceAudience classes such as WALKey and WALEdit. Will be removed in hbase-3.0.0.
*/
+ @Deprecated
default void postWALWrite(ObserverContext<? extends WALCoprocessorEnvironment> ctx,
RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
index 4cb9ed1..ee480be 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -96,14 +96,14 @@ public abstract class AbstractMemStore implements MemStore {
public abstract void updateLowestUnflushedSequenceIdInWAL(boolean onlyIfMoreRecent);
@Override
- public void add(Iterable<Cell> cells, MemStoreSize memstoreSize) {
+ public void add(Iterable<Cell> cells, MemStoreSizing memstoreSizing) {
for (Cell cell : cells) {
- add(cell, memstoreSize);
+ add(cell, memstoreSizing);
}
}
@Override
- public void add(Cell cell, MemStoreSize memstoreSize) {
+ public void add(Cell cell, MemStoreSizing memstoreSizing) {
Cell toAdd = maybeCloneWithAllocator(cell);
boolean mslabUsed = (toAdd != cell);
// This cell data is backed by the same byte[] where we read request in RPC(See HBASE-15180). By
@@ -118,7 +118,7 @@ public abstract class AbstractMemStore implements MemStore {
if (!mslabUsed) {
toAdd = deepCopyIfNeeded(toAdd);
}
- internalAdd(toAdd, mslabUsed, memstoreSize);
+ internalAdd(toAdd, mslabUsed, memstoreSizing);
}
private static Cell deepCopyIfNeeded(Cell cell) {
@@ -129,9 +129,9 @@ public abstract class AbstractMemStore implements MemStore {
}
@Override
- public void upsert(Iterable<Cell> cells, long readpoint, MemStoreSize memstoreSize) {
+ public void upsert(Iterable<Cell> cells, long readpoint, MemStoreSizing memstoreSizing) {
for (Cell cell : cells) {
- upsert(cell, readpoint, memstoreSize);
+ upsert(cell, readpoint, memstoreSizing);
}
}
@@ -167,7 +167,11 @@ public abstract class AbstractMemStore implements MemStore {
@Override
public MemStoreSize getSnapshotSize() {
- return new MemStoreSize(this.snapshot.keySize(), this.snapshot.heapSize());
+ return getSnapshotSizing();
+ }
+
+ MemStoreSizing getSnapshotSizing() {
+ return new MemStoreSizing(this.snapshot.keySize(), this.snapshot.heapSize());
}
@Override
@@ -210,7 +214,7 @@ public abstract class AbstractMemStore implements MemStore {
* @param readpoint readpoint below which we can safely remove duplicate KVs
* @param memstoreSize
*/
- private void upsert(Cell cell, long readpoint, MemStoreSize memstoreSize) {
+ private void upsert(Cell cell, long readpoint, MemStoreSizing memstoreSizing) {
// Add the Cell to the MemStore
// Use the internalAdd method here since we (a) already have a lock
// and (b) cannot safely use the MSLAB here without potentially
@@ -221,7 +225,7 @@ public abstract class AbstractMemStore implements MemStore {
// must do below deep copy. Or else we will keep referring to the bigger chunk of memory and
// prevent it from getting GCed.
cell = deepCopyIfNeeded(cell);
- this.active.upsert(cell, readpoint, memstoreSize);
+ this.active.upsert(cell, readpoint, memstoreSizing);
setOldestEditTimeToNow();
checkActiveSize();
}
@@ -277,8 +281,8 @@ public abstract class AbstractMemStore implements MemStore {
* @param mslabUsed whether using MSLAB
* @param memstoreSize
*/
- private void internalAdd(final Cell toAdd, final boolean mslabUsed, MemStoreSize memstoreSize) {
- active.add(toAdd, mslabUsed, memstoreSize);
+ private void internalAdd(final Cell toAdd, final boolean mslabUsed, MemStoreSizing memstoreSizing) {
+ active.add(toAdd, mslabUsed, memstoreSizing);
setOldestEditTimeToNow();
checkActiveSize();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java
index ea49fca..8cd5f2a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.regionserver;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
-import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.util.ClassSize;
@@ -55,7 +54,7 @@ public class CellArrayImmutableSegment extends ImmutableSegment {
* of CSLMImmutableSegment
* The given iterator returns the Cells that "survived" the compaction.
*/
- protected CellArrayImmutableSegment(CSLMImmutableSegment segment, MemStoreSize memstoreSize) {
+ protected CellArrayImmutableSegment(CSLMImmutableSegment segment, MemStoreSizing memstoreSizing) {
super(segment); // initiailize the upper class
incSize(0, DEEP_OVERHEAD_CAM - CSLMImmutableSegment.DEEP_OVERHEAD_CSLM);
int numOfCells = segment.getCellsCount();
@@ -65,7 +64,7 @@ public class CellArrayImmutableSegment extends ImmutableSegment {
// add sizes of CellArrayMap entry (reinitializeCellSet doesn't take the care for the sizes)
long newSegmentSizeDelta = numOfCells*(indexEntrySize()-ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY);
incSize(0, newSegmentSizeDelta);
- memstoreSize.incMemStoreSize(0, newSegmentSizeDelta);
+ memstoreSizing.incMemStoreSize(0, newSegmentSizeDelta);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
index b680b61..4ef0657 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.regionserver;
import org.apache.hadoop.hbase.ByteBufferKeyValue;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
-import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.yetus.audience.InterfaceAudience;
@@ -61,7 +60,8 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
* of CSLMImmutableSegment
* The given iterator returns the Cells that "survived" the compaction.
*/
- protected CellChunkImmutableSegment(CSLMImmutableSegment segment, MemStoreSize memstoreSize) {
+ protected CellChunkImmutableSegment(CSLMImmutableSegment segment,
+ MemStoreSizing memstoreSizing) {
super(segment); // initiailize the upper class
incSize(0,-CSLMImmutableSegment.DEEP_OVERHEAD_CSLM + CellChunkImmutableSegment.DEEP_OVERHEAD_CCM);
int numOfCells = segment.getCellsCount();
@@ -73,7 +73,7 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
long newSegmentSizeDelta = numOfCells*(indexEntrySize()-ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY);
incSize(0, newSegmentSizeDelta);
- memstoreSize.incMemStoreSize(0, newSegmentSizeDelta);
+ memstoreSizing.incMemStoreSize(0, newSegmentSizeDelta);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
index 01138df..e6f9451 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
@@ -146,12 +146,12 @@ public class CompactingMemStore extends AbstractMemStore {
*/
@Override
public MemStoreSize size() {
- MemStoreSize memstoreSize = new MemStoreSize();
- memstoreSize.incMemStoreSize(this.active.keySize(), this.active.heapSize());
+ MemStoreSizing memstoreSizing = new MemStoreSizing();
+ memstoreSizing.incMemStoreSize(this.active.keySize(), this.active.heapSize());
for (Segment item : pipeline.getSegments()) {
- memstoreSize.incMemStoreSize(item.keySize(), item.heapSize());
+ memstoreSizing.incMemStoreSize(item.keySize(), item.heapSize());
}
- return memstoreSize;
+ return memstoreSizing;
}
/**
@@ -215,17 +215,17 @@ public class CompactingMemStore extends AbstractMemStore {
*/
@Override
public MemStoreSize getFlushableSize() {
- MemStoreSize snapshotSize = getSnapshotSize();
- if (snapshotSize.getDataSize() == 0) {
+ MemStoreSizing snapshotSizing = getSnapshotSizing();
+ if (snapshotSizing.getDataSize() == 0) {
// if snapshot is empty the tail of the pipeline (or everything in the memstore) is flushed
if (compositeSnapshot) {
- snapshotSize = pipeline.getPipelineSize();
- snapshotSize.incMemStoreSize(this.active.keySize(), this.active.heapSize());
+ snapshotSizing = pipeline.getPipelineSizing();
+ snapshotSizing.incMemStoreSize(this.active.keySize(), this.active.heapSize());
} else {
- snapshotSize = pipeline.getTailSize();
+ snapshotSizing = pipeline.getTailSizing();
}
}
- return snapshotSize.getDataSize() > 0 ? snapshotSize
+ return snapshotSizing.getDataSize() > 0 ? snapshotSizing
: new MemStoreSize(this.active.keySize(), this.active.heapSize());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
index ccbfbe3..e44cb45 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -149,7 +149,7 @@ public class CompactionPipeline {
long newHeapSize = 0;
if(segment != null) newHeapSize = segment.heapSize();
long heapSizeDelta = suffixHeapSize - newHeapSize;
- region.addMemStoreSize(new MemStoreSize(-dataSizeDelta, -heapSizeDelta));
+ region.addMemStoreSize(new MemStoreSizing(-dataSizeDelta, -heapSizeDelta));
if (LOG.isDebugEnabled()) {
LOG.debug("Suffix data size: " + suffixDataSize + " new segment data size: "
+ newDataSize + ". Suffix heap size: " + suffixHeapSize
@@ -199,14 +199,14 @@ public class CompactionPipeline {
int i = 0;
for (ImmutableSegment s : pipeline) {
if ( s.canBeFlattened() ) {
- MemStoreSize newMemstoreSize = new MemStoreSize(); // the size to be updated
+ MemStoreSizing newMemstoreAccounting = new MemStoreSizing(); // the size to be updated
ImmutableSegment newS = SegmentFactory.instance().createImmutableSegmentByFlattening(
- (CSLMImmutableSegment)s,idxType,newMemstoreSize);
+ (CSLMImmutableSegment)s,idxType,newMemstoreAccounting);
replaceAtIndex(i,newS);
if(region != null) {
// update the global memstore size counter
// upon flattening there is no change in the data size
- region.addMemStoreSize(new MemStoreSize(0, newMemstoreSize.getHeapSize()));
+ region.addMemStoreSize(new MemStoreSize(0, newMemstoreAccounting.getHeapSize()));
}
LOG.debug("Compaction pipeline segment " + s + " was flattened");
return true;
@@ -241,22 +241,22 @@ public class CompactionPipeline {
return minSequenceId;
}
- public MemStoreSize getTailSize() {
+ public MemStoreSizing getTailSizing() {
LinkedList<? extends Segment> localCopy = readOnlyCopy;
- if (localCopy.isEmpty()) return new MemStoreSize(true);
- return new MemStoreSize(localCopy.peekLast().keySize(), localCopy.peekLast().heapSize());
+ if (localCopy.isEmpty()) return new MemStoreSizing();
+ return new MemStoreSizing(localCopy.peekLast().keySize(), localCopy.peekLast().heapSize());
}
- public MemStoreSize getPipelineSize() {
+ public MemStoreSizing getPipelineSizing() {
long keySize = 0;
long heapSize = 0;
LinkedList<? extends Segment> localCopy = readOnlyCopy;
- if (localCopy.isEmpty()) return new MemStoreSize(true);
+ if (localCopy.isEmpty()) return new MemStoreSizing();
for (Segment segment : localCopy) {
keySize += segment.keySize();
heapSize += segment.heapSize();
}
- return new MemStoreSize(keySize, heapSize);
+ return new MemStoreSizing(keySize, heapSize);
}
private void swapSuffix(List<? extends Segment> suffix, ImmutableSegment segment,
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java
index ef4c3cd..9365819 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java
@@ -257,13 +257,13 @@ public class CompositeImmutableSegment extends ImmutableSegment {
}
@Override
- protected void internalAdd(Cell cell, boolean mslabUsed, MemStoreSize memstoreSize) {
+ protected void internalAdd(Cell cell, boolean mslabUsed, MemStoreSizing memstoreSizing) {
throw new IllegalStateException("Not supported by CompositeImmutableScanner");
}
@Override
protected void updateMetaInfo(Cell cellToAdd, boolean succ, boolean mslabUsed,
- MemStoreSize memstoreSize) {
+ MemStoreSizing memstoreSizing) {
throw new IllegalStateException("Not supported by CompositeImmutableScanner");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
index 4ce6e5b..b1a87be 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
@@ -195,26 +195,26 @@ public class DefaultMemStore extends AbstractMemStore {
byte [] fam = Bytes.toBytes("col");
byte [] qf = Bytes.toBytes("umn");
byte [] empty = new byte[0];
- MemStoreSize memstoreSize = new MemStoreSize();
+ MemStoreSizing memstoreSizing = new MemStoreSizing();
for (int i = 0; i < count; i++) {
// Give each its own ts
- memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty), memstoreSize);
+ memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty), memstoreSizing);
}
LOG.info("memstore1 estimated size="
- + (memstoreSize.getDataSize() + memstoreSize.getHeapSize()));
+ + (memstoreSizing.getDataSize() + memstoreSizing.getHeapSize()));
for (int i = 0; i < count; i++) {
- memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty), memstoreSize);
+ memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty), memstoreSizing);
}
LOG.info("memstore1 estimated size (2nd loading of same data)="
- + (memstoreSize.getDataSize() + memstoreSize.getHeapSize()));
+ + (memstoreSizing.getDataSize() + memstoreSizing.getHeapSize()));
// Make a variably sized memstore.
DefaultMemStore memstore2 = new DefaultMemStore();
- memstoreSize = new MemStoreSize();
+ memstoreSizing = new MemStoreSizing();
for (int i = 0; i < count; i++) {
- memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, new byte[i]), memstoreSize);
+ memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, new byte[i]), memstoreSizing);
}
LOG.info("memstore2 estimated size="
- + (memstoreSize.getDataSize() + memstoreSize.getHeapSize()));
+ + (memstoreSizing.getDataSize() + memstoreSizing.getHeapSize()));
final int seconds = 30;
LOG.info("Waiting " + seconds + " seconds while heap dump is taken");
LOG.info("Exiting.");
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 99f5c35..e03e4dd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -537,11 +537,11 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
final long startTime;
final long flushOpSeqId;
final long flushedSeqId;
- final MemStoreSize totalFlushableSize;
+ final MemStoreSizing totalFlushableSize;
/** Constructs an early exit case */
PrepareFlushResult(FlushResultImpl result, long flushSeqId) {
- this(result, null, null, null, Math.max(0, flushSeqId), 0, 0, new MemStoreSize());
+ this(result, null, null, null, Math.max(0, flushSeqId), 0, 0, MemStoreSizing.DUD);
}
/** Constructs a successful prepare flush result */
@@ -549,7 +549,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
TreeMap<byte[], StoreFlushContext> storeFlushCtxs,
TreeMap<byte[], List<Path>> committedFiles,
TreeMap<byte[], MemStoreSize> storeFlushableSize, long startTime, long flushSeqId,
- long flushedSeqId, MemStoreSize totalFlushableSize) {
+ long flushedSeqId, MemStoreSizing totalFlushableSize) {
this(null, storeFlushCtxs, committedFiles, storeFlushableSize, startTime,
flushSeqId, flushedSeqId, totalFlushableSize);
}
@@ -559,7 +559,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
TreeMap<byte[], StoreFlushContext> storeFlushCtxs,
TreeMap<byte[], List<Path>> committedFiles,
TreeMap<byte[], MemStoreSize> storeFlushableSize, long startTime, long flushSeqId,
- long flushedSeqId, MemStoreSize totalFlushableSize) {
+ long flushedSeqId, MemStoreSizing totalFlushableSize) {
this.result = result;
this.storeFlushCtxs = storeFlushCtxs;
this.committedFiles = committedFiles;
@@ -1711,7 +1711,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
this.closed.set(true);
if (!canFlush) {
- this.decrMemStoreSize(new MemStoreSize(memstoreDataSize.get(), getMemStoreHeapSize()));
+ this.decrMemStoreSize(new MemStoreSizing(memstoreDataSize.get(), getMemStoreHeapSize()));
} else if (memstoreDataSize.get() != 0) {
LOG.error("Memstore size is " + memstoreDataSize.get());
}
@@ -2502,7 +2502,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// block waiting for the lock for internal flush
this.updatesLock.writeLock().lock();
status.setStatus("Preparing flush snapshotting stores in " + getRegionInfo().getEncodedName());
- MemStoreSize totalSizeOfFlushableStores = new MemStoreSize();
+ MemStoreSizing totalSizeOfFlushableStores = new MemStoreSizing();
Map<byte[], Long> flushedFamilyNamesToSeq = new HashMap<>();
for (HStore store : storesToFlush) {
@@ -2546,7 +2546,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
for (HStore s : storesToFlush) {
MemStoreSize flushableSize = s.getFlushableSize();
totalSizeOfFlushableStores.incMemStoreSize(flushableSize);
- storeFlushCtxs.put(s.getColumnFamilyDescriptor().getName(), s.createFlushContext(flushOpSeqId));
+ storeFlushCtxs.put(s.getColumnFamilyDescriptor().getName(),
+ s.createFlushContext(flushOpSeqId));
committedFiles.put(s.getColumnFamilyDescriptor().getName(), null); // for writing stores to WAL
storeFlushableSize.put(s.getColumnFamilyDescriptor().getName(), flushableSize);
}
@@ -3323,7 +3324,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
int cellCount = 0;
/** Keep track of the locks we hold so we can release them in finally clause */
List<RowLock> acquiredRowLocks = Lists.newArrayListWithCapacity(batchOp.operations.length);
- MemStoreSize memStoreSize = new MemStoreSize();
+ MemStoreSizing memStoreAccounting = new MemStoreSizing();
try {
// STEP 1. Try to acquire as many locks as we can, and ensure we acquire at least one.
int numReadyToWrite = 0;
@@ -3506,11 +3507,11 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
this.updateSequenceId(batchOp.familyCellMaps[i].values(),
replay? batchOp.getReplaySequenceId(): writeEntry.getWriteNumber());
}
- applyFamilyMapToMemStore(batchOp.familyCellMaps[i], memStoreSize);
+ applyFamilyMapToMemStore(batchOp.familyCellMaps[i], memStoreAccounting);
}
// update memstore size
- this.addAndGetMemStoreSize(memStoreSize);
+ this.addAndGetMemStoreSize(memStoreAccounting);
// calling the post CP hook for batch mutation
if (!replay && coprocessorHost != null) {
@@ -3983,12 +3984,12 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
* @param memstoreSize
*/
private void applyFamilyMapToMemStore(Map<byte[], List<Cell>> familyMap,
- MemStoreSize memstoreSize) throws IOException {
+ MemStoreSizing memstoreAccounting) throws IOException {
for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) {
byte[] family = e.getKey();
List<Cell> cells = e.getValue();
assert cells instanceof RandomAccess;
- applyToMemStore(getStore(family), cells, false, memstoreSize);
+ applyToMemStore(getStore(family), cells, false, memstoreAccounting);
}
}
@@ -3996,30 +3997,30 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
* @param delta If we are doing delta changes -- e.g. increment/append -- then this flag will be
* set; when set we will run operations that make sense in the increment/append scenario
* but that do not make sense otherwise.
- * @see #applyToMemStore(HStore, Cell, MemStoreSize)
+ * @see #applyToMemStore(HStore, Cell, MemStoreSizing)
*/
private void applyToMemStore(HStore store, List<Cell> cells, boolean delta,
- MemStoreSize memstoreSize) throws IOException {
+ MemStoreSizing memstoreAccounting) throws IOException {
// Any change in how we update Store/MemStore needs to also be done in other applyToMemStore!!!!
boolean upsert = delta && store.getColumnFamilyDescriptor().getMaxVersions() == 1;
if (upsert) {
- store.upsert(cells, getSmallestReadPoint(), memstoreSize);
+ store.upsert(cells, getSmallestReadPoint(), memstoreAccounting);
} else {
- store.add(cells, memstoreSize);
+ store.add(cells, memstoreAccounting);
}
}
/**
- * @see #applyToMemStore(HStore, List, boolean, MemStoreSize)
+ * @see #applyToMemStore(HStore, List, boolean, MemStoreSizing)
*/
- private void applyToMemStore(HStore store, Cell cell, MemStoreSize memstoreSize)
+ private void applyToMemStore(HStore store, Cell cell, MemStoreSizing memstoreAccounting)
throws IOException {
// Any change in how we update Store/MemStore needs to also be done in other applyToMemStore!!!!
if (store == null) {
checkFamily(CellUtil.cloneFamily(cell));
// Unreachable because checkFamily will throw exception
}
- store.add(cell, memstoreSize);
+ store.add(cell, memstoreAccounting);
}
/**
@@ -4347,7 +4348,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
}
boolean flush = false;
- MemStoreSize memstoreSize = new MemStoreSize();
+ MemStoreSizing memstoreSize = new MemStoreSizing();
for (Cell cell: val.getCells()) {
// Check this edit is for me. Also, guard against writing the special
// METACOLUMN info such as HBASE::CACHEFLUSH entries
@@ -4843,7 +4844,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
* @throws IOException
*/
private MemStoreSize dropMemStoreContentsForSeqId(long seqId, HStore store) throws IOException {
- MemStoreSize totalFreedSize = new MemStoreSize();
+ MemStoreSizing totalFreedSize = new MemStoreSizing();
this.updatesLock.writeLock().lock();
try {
@@ -5271,15 +5272,13 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
* Used by tests
* @param s Store to add edit too.
* @param cell Cell to add.
- * @param memstoreSize
*/
@VisibleForTesting
- protected void restoreEdit(HStore s, Cell cell, MemStoreSize memstoreSize) {
- s.add(cell, memstoreSize);
+ protected void restoreEdit(HStore s, Cell cell, MemStoreSizing memstoreAccounting) {
+ s.add(cell, memstoreAccounting);
}
/**
- * @param fs
* @param p File to check.
* @return True if file was zero-length (and if so, we'll delete it in here).
* @throws IOException
@@ -7114,7 +7113,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// This is assigned by mvcc either explicity in the below or in the guts of the WAL append
// when it assigns the edit a sequencedid (A.K.A the mvcc write number).
WriteEntry writeEntry = null;
- MemStoreSize memstoreSize = new MemStoreSize();
+ MemStoreSizing memstoreAccounting = new MemStoreSizing();
try {
boolean success = false;
try {
@@ -7158,7 +7157,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// If no WAL, need to stamp it here.
CellUtil.setSequenceId(cell, sequenceId);
}
- applyToMemStore(getStore(cell), cell, memstoreSize);
+ applyToMemStore(getStore(cell), cell, memstoreAccounting);
}
}
@@ -7194,7 +7193,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
} finally {
closeRegionOperation();
if (!mutations.isEmpty()) {
- long newSize = this.addAndGetMemStoreSize(memstoreSize);
+ long newSize = this.addAndGetMemStoreSize(memstoreAccounting);
requestFlushIfNeeded(newSize);
}
}
@@ -7298,7 +7297,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
startRegionOperation(op);
List<Cell> results = returnResults? new ArrayList<>(mutation.size()): null;
RowLock rowLock = null;
- MemStoreSize memstoreSize = new MemStoreSize();
+ MemStoreSizing memstoreAccounting = new MemStoreSizing();
try {
rowLock = getRowLockInternal(mutation.getRow(), false);
lock(this.updatesLock.readLock());
@@ -7324,7 +7323,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
}
// Now write to MemStore. Do it a column family at a time.
for (Map.Entry<HStore, List<Cell>> e : forMemStore.entrySet()) {
- applyToMemStore(e.getKey(), e.getValue(), true, memstoreSize);
+ applyToMemStore(e.getKey(), e.getValue(), true, memstoreAccounting);
}
mvcc.completeAndWait(writeEntry);
if (rsServices != null && rsServices.getNonceManager() != null) {
@@ -7347,7 +7346,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
rowLock.release();
}
// Request a cache flush if over the limit. Do it outside update lock.
- if (isFlushSize(addAndGetMemStoreSize(memstoreSize))) {
+ if (isFlushSize(addAndGetMemStoreSize(memstoreAccounting))) {
requestFlush();
}
closeRegionOperation(op);
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 2ec5437..e3d6724 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -682,13 +682,11 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
/**
* Adds a value to the memstore
- * @param cell
- * @param memstoreSize
*/
- public void add(final Cell cell, MemStoreSize memstoreSize) {
+ public void add(final Cell cell, MemStoreSizing memstoreSizing) {
lock.readLock().lock();
try {
- this.memstore.add(cell, memstoreSize);
+ this.memstore.add(cell, memstoreSizing);
} finally {
lock.readLock().unlock();
}
@@ -696,13 +694,11 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
/**
* Adds the specified value to the memstore
- * @param cells
- * @param memstoreSize
*/
- public void add(final Iterable<Cell> cells, MemStoreSize memstoreSize) {
+ public void add(final Iterable<Cell> cells, MemStoreSizing memstoreSizing) {
lock.readLock().lock();
try {
- memstore.add(cells, memstoreSize);
+ memstore.add(cells, memstoreSizing);
} finally {
lock.readLock().unlock();
}
@@ -2143,16 +2139,14 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
* <p>
* This operation is atomic on each KeyValue (row/family/qualifier) but not necessarily atomic
* across all of them.
- * @param cells
* @param readpoint readpoint below which we can safely remove duplicate KVs
- * @param memstoreSize
* @throws IOException
*/
- public void upsert(Iterable<Cell> cells, long readpoint, MemStoreSize memstoreSize)
+ public void upsert(Iterable<Cell> cells, long readpoint, MemStoreSizing memstoreSizing)
throws IOException {
this.lock.readLock().lock();
try {
- this.memstore.upsert(cells, readpoint, memstoreSize);
+ this.memstore.upsert(cells, readpoint, memstoreSizing);
} finally {
this.lock.readLock().unlock();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
index 324afbe..af7e5d5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -68,18 +68,18 @@ public interface MemStore {
/**
* Write an update
* @param cell
- * @param memstoreSize The delta in memstore size will be passed back via this.
+ * @param memstoreSizing The delta in memstore size will be passed back via this.
* This will include both data size and heap overhead delta.
*/
- void add(final Cell cell, MemStoreSize memstoreSize);
+ void add(final Cell cell, MemStoreSizing memstoreSizing);
/**
* Write the updates
* @param cells
- * @param memstoreSize The delta in memstore size will be passed back via this.
+ * @param memstoreSizing The delta in memstore size will be passed back via this.
* This will include both data size and heap overhead delta.
*/
- void add(Iterable<Cell> cells, MemStoreSize memstoreSize);
+ void add(Iterable<Cell> cells, MemStoreSizing memstoreSizing);
/**
* @return Oldest timestamp of all the Cells in the MemStore
@@ -99,10 +99,10 @@ public interface MemStore {
* only see each KeyValue update as atomic.
* @param cells
* @param readpoint readpoint below which we can safely remove duplicate Cells.
- * @param memstoreSize The delta in memstore size will be passed back via this.
+ * @param memstoreSizing The delta in memstore size will be passed back via this.
* This will include both data size and heap overhead delta.
*/
- void upsert(Iterable<Cell> cells, long readpoint, MemStoreSize memstoreSize);
+ void upsert(Iterable<Cell> cells, long readpoint, MemStoreSizing memstoreSizing);
/**
* @return scanner over the memstore. This might include scanner over the snapshot when one is
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java
index a588b20..cf2ef6f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,71 +17,47 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.yetus.audience.InterfaceAudience;
/**
- * Wraps the data size part and total heap space occupied by the memstore.
+ * Reports the data size part and total heap space occupied by the MemStore.
+ * Read-only.
+ * @see MemStoreSizing
*/
-@InterfaceAudience.Private
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
public class MemStoreSize {
-
- // 'dataSize' tracks the Cell's data bytes size alone (Key bytes, value bytes). A cell's data can
- // be in on heap or off heap area depending on the MSLAB and its configuration to be using on heap
- // or off heap LABs
- private long dataSize;
- // 'heapSize' tracks all Cell's heap size occupancy. This will include Cell POJO heap overhead.
- // When Cells in on heap area, this will include the cells data size as well.
- private long heapSize;
- final private boolean isEmpty;
+ /**
+ *'dataSize' tracks the Cell's data bytes size alone (Key bytes, value bytes). A cell's data can
+ * be in on heap or off heap area depending on the MSLAB and its configuration to be using on heap
+ * or off heap LABs
+ */
+ protected long dataSize;
+
+ /** 'heapSize' tracks all Cell's heap size occupancy. This will include Cell POJO heap overhead.
+ * When Cells in on heap area, this will include the cells data size as well.
+ */
+ protected long heapSize;
public MemStoreSize() {
- dataSize = 0;
- heapSize = 0;
- isEmpty = false;
- }
-
- public MemStoreSize(boolean isEmpty) {
- dataSize = 0;
- heapSize = 0;
- this.isEmpty = isEmpty;
- }
-
- public boolean isEmpty() {
- return isEmpty;
+ this(0L, 0L);
}
public MemStoreSize(long dataSize, long heapSize) {
this.dataSize = dataSize;
this.heapSize = heapSize;
- this.isEmpty = false;
}
- public void incMemStoreSize(long dataSizeDelta, long heapSizeDelta) {
- this.dataSize += dataSizeDelta;
- this.heapSize += heapSizeDelta;
- }
-
- public void incMemStoreSize(MemStoreSize delta) {
- this.dataSize += delta.dataSize;
- this.heapSize += delta.heapSize;
- }
-
- public void decMemStoreSize(long dataSizeDelta, long heapSizeDelta) {
- this.dataSize -= dataSizeDelta;
- this.heapSize -= heapSizeDelta;
- }
-
- public void decMemStoreSize(MemStoreSize delta) {
- this.dataSize -= delta.dataSize;
- this.heapSize -= delta.heapSize;
+ public boolean isEmpty() {
+ return this.dataSize == 0 && this.heapSize == 0;
}
public long getDataSize() {
- return isEmpty ? 0 : dataSize;
+ return this.dataSize;
}
public long getHeapSize() {
- return isEmpty ? 0 : heapSize;
+ return this.heapSize;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java
new file mode 100644
index 0000000..fade622
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Accounting of current heap and data sizes.
+ * Allows read/write on data/heap size as opposed to {@Link MemStoreSize} which is read-only.
+ * For internal use.
+ * @see MemStoreSize
+ */
+@InterfaceAudience.Private
+public class MemStoreSizing extends MemStoreSize {
+ public static final MemStoreSizing DUD = new MemStoreSizing() {
+ @Override
+ public void incMemStoreSize(MemStoreSize delta) {
+ incMemStoreSize(delta.getDataSize(), delta.getHeapSize());
+ }
+
+ @Override
+ public void incMemStoreSize(long dataSizeDelta, long heapSizeDelta) {
+ throw new RuntimeException("I'm a dud, you can't use me!");
+ }
+
+ @Override
+ public void decMemStoreSize(MemStoreSize delta) {
+ decMemStoreSize(delta.getDataSize(), delta.getHeapSize());
+ }
+
+ @Override
+ public void decMemStoreSize(long dataSizeDelta, long heapSizeDelta) {
+ throw new RuntimeException("I'm a dud, you can't use me!");
+ }
+ };
+
+ public MemStoreSizing() {
+ super();
+ }
+
+ public MemStoreSizing(long dataSize, long heapSize) {
+ super(dataSize, heapSize);
+ }
+
+ public void incMemStoreSize(long dataSizeDelta, long heapSizeDelta) {
+ this.dataSize += dataSizeDelta;
+ this.heapSize += heapSizeDelta;
+ }
+
+ public void incMemStoreSize(MemStoreSize delta) {
+ incMemStoreSize(delta.getDataSize(), delta.getHeapSize());
+ }
+
+ public void decMemStoreSize(long dataSizeDelta, long heapSizeDelta) {
+ this.dataSize -= dataSizeDelta;
+ this.heapSize -= heapSizeDelta;
+ }
+
+ public void decMemStoreSize(MemStoreSize delta) {
+ decMemStoreSize(delta.getDataSize(), delta.getHeapSize());
+ }
+
+ public void empty() {
+ this.dataSize = 0L;
+ this.heapSize = 0L;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || !(obj instanceof MemStoreSizing)) {
+ return false;
+ }
+ MemStoreSizing other = (MemStoreSizing) obj;
+ return this.dataSize == other.dataSize && this.heapSize == other.heapSize;
+ }
+
+ @Override
+ public int hashCode() {
+ long h = 13 * this.dataSize;
+ h = h + 14 * this.heapSize;
+ return (int) h;
+ }
+
+ @Override
+ public String toString() {
+ return "dataSize=" + this.dataSize + " , heapSize=" + this.heapSize;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java
index c04e41e..56a97e0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MiniBatchOperationInProgress.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.wal.WALEdit;
@@ -30,7 +31,7 @@ import org.apache.hadoop.hbase.wal.WALEdit;
* org.apache.hadoop.hbase.coprocessor.ObserverContext, MiniBatchOperationInProgress)
* @param T Pair<Mutation, Integer> pair of Mutations and associated rowlock ids .
*/
-@InterfaceAudience.LimitedPrivate("Coprocessors")
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
public class MiniBatchOperationInProgress<T> {
private final T[] operations;
private Mutation[][] operationsFromCoprocessors;
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
index b1dc748..68c0fa1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -27,7 +27,6 @@ import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.io.TimeRange;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
@@ -51,14 +50,13 @@ public class MutableSegment extends Segment {
* Adds the given cell into the segment
* @param cell the cell to add
* @param mslabUsed whether using MSLAB
- * @param memstoreSize
*/
- public void add(Cell cell, boolean mslabUsed, MemStoreSize memstoreSize) {
- internalAdd(cell, mslabUsed, memstoreSize);
+ public void add(Cell cell, boolean mslabUsed, MemStoreSizing memStoreSizing) {
+ internalAdd(cell, mslabUsed, memStoreSizing);
}
- public void upsert(Cell cell, long readpoint, MemStoreSize memstoreSize) {
- internalAdd(cell, false, memstoreSize);
+ public void upsert(Cell cell, long readpoint, MemStoreSizing memStoreSizing) {
+ internalAdd(cell, false, memStoreSizing);
// Get the Cells for the row/family/qualifier regardless of timestamp.
// For this case we want to clean up any other puts
@@ -90,8 +88,8 @@ public class MutableSegment extends Segment {
int cellLen = getCellLength(cur);
long heapSize = heapSizeChange(cur, true);
this.incSize(-cellLen, -heapSize);
- if (memstoreSize != null) {
- memstoreSize.decMemStoreSize(cellLen, heapSize);
+ if (memStoreSizing != null) {
+ memStoreSizing.decMemStoreSize(cellLen, heapSize);
}
it.remove();
} else {
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
index 8d1656f..7689fcd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.util.Pair;
*/
@InterfaceAudience.Private
public class RegionServerAccounting {
-
// memstore data size
private final LongAdder globalMemstoreDataSize = new LongAdder();
// memstore heap size. When off heap MSLAB in place, this will be only heap overhead of the Cell
@@ -46,7 +45,7 @@ public class RegionServerAccounting {
// Store the edits size during replaying WAL. Use this to roll back the
// global memstore size once a region opening failed.
- private final ConcurrentMap<byte[], MemStoreSize> replayEditsPerRegion =
+ private final ConcurrentMap<byte[], MemStoreSizing> replayEditsPerRegion =
new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
private long globalMemStoreLimit;
@@ -216,14 +215,14 @@ public class RegionServerAccounting {
* @param memStoreSize the Memstore size will be added to replayEditsPerRegion.
*/
public void addRegionReplayEditsSize(byte[] regionName, MemStoreSize memStoreSize) {
- MemStoreSize replayEdistsSize = replayEditsPerRegion.get(regionName);
+ MemStoreSizing replayEdistsSize = replayEditsPerRegion.get(regionName);
// All ops on the same MemStoreSize object is going to be done by single thread, sequentially
// only. First calls to this method to increment the per region reply edits size and then call
// to either rollbackRegionReplayEditsSize or clearRegionReplayEditsSize as per the result of
// the region open operation. No need to handle multi thread issues on one region's entry in
// this Map.
if (replayEdistsSize == null) {
- replayEdistsSize = new MemStoreSize();
+ replayEdistsSize = new MemStoreSizing();
replayEditsPerRegion.put(regionName, replayEdistsSize);
}
replayEdistsSize.incMemStoreSize(memStoreSize);
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
index 03145f1..6f41f86 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -277,13 +277,13 @@ public abstract class Segment {
return comparator;
}
- protected void internalAdd(Cell cell, boolean mslabUsed, MemStoreSize memstoreSize) {
+ protected void internalAdd(Cell cell, boolean mslabUsed, MemStoreSizing memstoreSizing) {
boolean succ = getCellSet().add(cell);
- updateMetaInfo(cell, succ, mslabUsed, memstoreSize);
+ updateMetaInfo(cell, succ, mslabUsed, memstoreSizing);
}
protected void updateMetaInfo(Cell cellToAdd, boolean succ, boolean mslabUsed,
- MemStoreSize memstoreSize) {
+ MemStoreSizing memstoreSizing) {
long cellSize = 0;
// If there's already a same cell in the CellSet and we are using MSLAB, we must count in the
// MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger
@@ -293,8 +293,8 @@ public abstract class Segment {
}
long heapSize = heapSizeChange(cellToAdd, succ);
incSize(cellSize, heapSize);
- if (memstoreSize != null) {
- memstoreSize.incMemStoreSize(cellSize, heapSize);
+ if (memstoreSizing != null) {
+ memstoreSizing.incMemStoreSize(cellSize, heapSize);
}
getTimeRangeTracker().includeTimestamp(cellToAdd);
minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId());
@@ -307,8 +307,8 @@ public abstract class Segment {
}
}
- protected void updateMetaInfo(Cell cellToAdd, boolean succ, MemStoreSize memstoreSize) {
- updateMetaInfo(cellToAdd, succ, (getMemStoreLAB()!=null), memstoreSize);
+ protected void updateMetaInfo(Cell cellToAdd, boolean succ, MemStoreSizing memstoreSizing) {
+ updateMetaInfo(cellToAdd, succ, (getMemStoreLAB()!=null), memstoreSizing);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java
index 63d1baa..43836f4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java
@@ -95,17 +95,18 @@ public final class SegmentFactory {
// create flat immutable segment from non-flat immutable segment
// for flattening
public ImmutableSegment createImmutableSegmentByFlattening(
- CSLMImmutableSegment segment, CompactingMemStore.IndexType idxType, MemStoreSize memstoreSize) {
+ CSLMImmutableSegment segment, CompactingMemStore.IndexType idxType,
+ MemStoreSizing memstoreSizing) {
ImmutableSegment res = null;
switch (idxType) {
case CHUNK_MAP:
- res = new CellChunkImmutableSegment(segment, memstoreSize);
+ res = new CellChunkImmutableSegment(segment, memstoreSizing);
break;
case CSLM_MAP:
assert false; // non-flat segment can not be the result of flattening
break;
case ARRAY_MAP:
- res = new CellArrayImmutableSegment(segment, memstoreSize);
+ res = new CellArrayImmutableSegment(segment, memstoreSizing);
break;
}
return res;
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
index 12b94b5..ce0075e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
@@ -607,18 +607,18 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
private long addRowsByKeys(final AbstractMemStore hmc, String[] keys) {
byte[] fam = Bytes.toBytes("testfamily");
byte[] qf = Bytes.toBytes("testqualifier");
- MemStoreSize memstoreSize = new MemStoreSize();
+ MemStoreSizing memstoreSizing = new MemStoreSizing();
for (int i = 0; i < keys.length; i++) {
long timestamp = System.currentTimeMillis();
Threads.sleep(1); // to make sure each kv gets a different ts
byte[] row = Bytes.toBytes(keys[i]);
byte[] val = Bytes.toBytes(keys[i] + i);
KeyValue kv = new KeyValue(row, fam, qf, timestamp, val);
- hmc.add(kv, memstoreSize);
+ hmc.add(kv, memstoreSizing);
LOG.debug("added kv: " + kv.getKeyString() + ", timestamp" + kv.getTimestamp());
}
- regionServicesForStores.addMemStoreSize(memstoreSize);
- return memstoreSize.getDataSize();
+ regionServicesForStores.addMemStoreSize(memstoreSizing);
+ return memstoreSizing.getDataSize();
}
private long cellBeforeFlushSize() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
index 0a974ae..6197a08 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
@@ -126,9 +126,9 @@ public class TestDefaultMemStore {
public void testPutSameCell() {
byte[] bytes = Bytes.toBytes(getName());
KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
- MemStoreSize sizeChangeForFirstCell = new MemStoreSize();
+ MemStoreSizing sizeChangeForFirstCell = new MemStoreSizing();
this.memstore.add(kv, sizeChangeForFirstCell);
- MemStoreSize sizeChangeForSecondCell = new MemStoreSize();
+ MemStoreSizing sizeChangeForSecondCell = new MemStoreSizing();
this.memstore.add(kv, sizeChangeForSecondCell);
// make sure memstore size increase won't double-count MSLAB chunk size
assertEquals(Segment.getCellLength(kv), sizeChangeForFirstCell.getDataSize());
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
index eeed73c..b9054f4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
@@ -230,7 +230,7 @@ public class TestHStore {
* @throws Exception
*/
@Test
- public void testFlushSizeAccounting() throws Exception {
+ public void testFlushSizeSizing() throws Exception {
LOG.info("Setting up a faulty file system that cannot write in " +
this.name.getMethodName());
final Configuration conf = HBaseConfiguration.create();
@@ -254,7 +254,7 @@ public class TestHStore {
MemStoreSize size = store.memstore.getFlushableSize();
assertEquals(0, size.getDataSize());
LOG.info("Adding some data");
- MemStoreSize kvSize = new MemStoreSize();
+ MemStoreSizing kvSize = new MemStoreSizing();
store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), kvSize);
// add the heap size of active (mutable) segment
kvSize.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD);
@@ -273,7 +273,7 @@ public class TestHStore {
CSLMImmutableSegment.DEEP_OVERHEAD_CSLM-MutableSegment.DEEP_OVERHEAD);
size = store.memstore.getFlushableSize();
assertEquals(kvSize, size);
- MemStoreSize kvSize2 = new MemStoreSize();
+ MemStoreSizing kvSize2 = new MemStoreSizing();
store.add(new KeyValue(row, family, qf2, 2, (byte[])null), kvSize2);
kvSize2.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD);
// Even though we add a new kv, we expect the flushable size to be 'same' since we have
@@ -1217,7 +1217,7 @@ public class TestHStore {
byte[] value0 = Bytes.toBytes("value0");
byte[] value1 = Bytes.toBytes("value1");
byte[] value2 = Bytes.toBytes("value2");
- MemStoreSize memStoreSize = new MemStoreSize();
+ MemStoreSizing memStoreSizing = new MemStoreSizing();
long ts = EnvironmentEdgeManager.currentTime();
long seqId = 100;
init(name.getMethodName(), conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)),
@@ -1229,18 +1229,18 @@ public class TestHStore {
}
});
// The cells having the value0 won't be flushed to disk because the value of max version is 1
- store.add(createCell(r0, qf1, ts, seqId, value0), memStoreSize);
- store.add(createCell(r0, qf2, ts, seqId, value0), memStoreSize);
- store.add(createCell(r0, qf3, ts, seqId, value0), memStoreSize);
- store.add(createCell(r1, qf1, ts + 1, seqId + 1, value1), memStoreSize);
- store.add(createCell(r1, qf2, ts + 1, seqId + 1, value1), memStoreSize);
- store.add(createCell(r1, qf3, ts + 1, seqId + 1, value1), memStoreSize);
- store.add(createCell(r2, qf1, ts + 2, seqId + 2, value2), memStoreSize);
- store.add(createCell(r2, qf2, ts + 2, seqId + 2, value2), memStoreSize);
- store.add(createCell(r2, qf3, ts + 2, seqId + 2, value2), memStoreSize);
- store.add(createCell(r1, qf1, ts + 3, seqId + 3, value1), memStoreSize);
- store.add(createCell(r1, qf2, ts + 3, seqId + 3, value1), memStoreSize);
- store.add(createCell(r1, qf3, ts + 3, seqId + 3, value1), memStoreSize);
+ store.add(createCell(r0, qf1, ts, seqId, value0), memStoreSizing);
+ store.add(createCell(r0, qf2, ts, seqId, value0), memStoreSizing);
+ store.add(createCell(r0, qf3, ts, seqId, value0), memStoreSizing);
+ store.add(createCell(r1, qf1, ts + 1, seqId + 1, value1), memStoreSizing);
+ store.add(createCell(r1, qf2, ts + 1, seqId + 1, value1), memStoreSizing);
+ store.add(createCell(r1, qf3, ts + 1, seqId + 1, value1), memStoreSizing);
+ store.add(createCell(r2, qf1, ts + 2, seqId + 2, value2), memStoreSizing);
+ store.add(createCell(r2, qf2, ts + 2, seqId + 2, value2), memStoreSizing);
+ store.add(createCell(r2, qf3, ts + 2, seqId + 2, value2), memStoreSizing);
+ store.add(createCell(r1, qf1, ts + 3, seqId + 3, value1), memStoreSizing);
+ store.add(createCell(r1, qf2, ts + 3, seqId + 3, value1), memStoreSizing);
+ store.add(createCell(r1, qf3, ts + 3, seqId + 3, value1), memStoreSizing);
List<Cell> myList = new MyList<>(hook);
Scan scan = new Scan()
.withStartRow(r1)
@@ -1276,13 +1276,13 @@ public class TestHStore {
init(name.getMethodName(), conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
.setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
byte[] value = Bytes.toBytes("value");
- MemStoreSize memStoreSize = new MemStoreSize();
+ MemStoreSizing memStoreSizing = new MemStoreSizing();
long ts = EnvironmentEdgeManager.currentTime();
long seqId = 100;
// older data whihc shouldn't be "seen" by client
- store.add(createCell(qf1, ts, seqId, value), memStoreSize);
- store.add(createCell(qf2, ts, seqId, value), memStoreSize);
- store.add(createCell(qf3, ts, seqId, value), memStoreSize);
+ store.add(createCell(qf1, ts, seqId, value), memStoreSizing);
+ store.add(createCell(qf2, ts, seqId, value), memStoreSizing);
+ store.add(createCell(qf3, ts, seqId, value), memStoreSizing);
TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR);
quals.add(qf1);
quals.add(qf2);
@@ -1354,22 +1354,22 @@ public class TestHStore {
});
byte[] oldValue = Bytes.toBytes("oldValue");
byte[] currentValue = Bytes.toBytes("currentValue");
- MemStoreSize memStoreSize = new MemStoreSize();
+ MemStoreSizing memStoreSizing = new MemStoreSizing();
long ts = EnvironmentEdgeManager.currentTime();
long seqId = 100;
// older data whihc shouldn't be "seen" by client
- myStore.add(createCell(qf1, ts, seqId, oldValue), memStoreSize);
- myStore.add(createCell(qf2, ts, seqId, oldValue), memStoreSize);
- myStore.add(createCell(qf3, ts, seqId, oldValue), memStoreSize);
+ myStore.add(createCell(qf1, ts, seqId, oldValue), memStoreSizing);
+ myStore.add(createCell(qf2, ts, seqId, oldValue), memStoreSizing);
+ myStore.add(createCell(qf3, ts, seqId, oldValue), memStoreSizing);
long snapshotId = id++;
// push older data into snapshot -- phase (1/4)
StoreFlushContext storeFlushCtx = store.createFlushContext(snapshotId);
storeFlushCtx.prepare();
// insert current data into active -- phase (2/4)
- myStore.add(createCell(qf1, ts + 1, seqId + 1, currentValue), memStoreSize);
- myStore.add(createCell(qf2, ts + 1, seqId + 1, currentValue), memStoreSize);
- myStore.add(createCell(qf3, ts + 1, seqId + 1, currentValue), memStoreSize);
+ myStore.add(createCell(qf1, ts + 1, seqId + 1, currentValue), memStoreSizing);
+ myStore.add(createCell(qf2, ts + 1, seqId + 1, currentValue), memStoreSizing);
+ myStore.add(createCell(qf3, ts + 1, seqId + 1, currentValue), memStoreSizing);
TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR);
quals.add(qf1);
quals.add(qf2);
@@ -1467,21 +1467,21 @@ public class TestHStore {
init(name.getMethodName(), conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
.setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
byte[] value = Bytes.toBytes("thisisavarylargevalue");
- MemStoreSize memStoreSize = new MemStoreSize();
+ MemStoreSizing memStoreSizing = new MemStoreSizing();
long ts = EnvironmentEdgeManager.currentTime();
long seqId = 100;
// older data whihc shouldn't be "seen" by client
- store.add(createCell(qf1, ts, seqId, value), memStoreSize);
- store.add(createCell(qf2, ts, seqId, value), memStoreSize);
- store.add(createCell(qf3, ts, seqId, value), memStoreSize);
+ store.add(createCell(qf1, ts, seqId, value), memStoreSizing);
+ store.add(createCell(qf2, ts, seqId, value), memStoreSizing);
+ store.add(createCell(qf3, ts, seqId, value), memStoreSizing);
assertEquals(1, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get());
StoreFlushContext storeFlushCtx = store.createFlushContext(id++);
storeFlushCtx.prepare();
// This shouldn't invoke another in-memory flush because the first compactor thread
// hasn't accomplished the in-memory compaction.
- store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSize);
- store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSize);
- store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSize);
+ store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing);
+ store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing);
+ store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing);
assertEquals(1, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get());
//okay. Let the compaction be completed
MyMemStoreCompactor.START_COMPACTOR_LATCH.countDown();
@@ -1490,9 +1490,9 @@ public class TestHStore {
TimeUnit.SECONDS.sleep(1);
}
// This should invoke another in-memory flush.
- store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSize);
- store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSize);
- store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSize);
+ store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing);
+ store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing);
+ store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing);
assertEquals(2, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get());
conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
String.valueOf(TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE));
@@ -1589,25 +1589,25 @@ public class TestHStore {
conf.setLong(StoreScanner.STORESCANNER_PREAD_MAX_BYTES, 0);
// Set the lower threshold to invoke the "MERGE" policy
MyStore store = initMyStore(name.getMethodName(), conf, new MyStoreHook() {});
- MemStoreSize memStoreSize = new MemStoreSize();
+ MemStoreSizing memStoreSizing = new MemStoreSizing();
long ts = System.currentTimeMillis();
long seqID = 1l;
// Add some data to the region and do some flushes
for (int i = 1; i < 10; i++) {
store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")),
- memStoreSize);
+ memStoreSizing);
}
// flush them
flushStore(store, seqID);
for (int i = 11; i < 20; i++) {
store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")),
- memStoreSize);
+ memStoreSizing);
}
// flush them
flushStore(store, seqID);
for (int i = 21; i < 30; i++) {
store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")),
- memStoreSize);
+ memStoreSizing);
}
// flush them
flushStore(store, seqID);
@@ -1624,14 +1624,14 @@ public class TestHStore {
// create more store files
for (int i = 31; i < 40; i++) {
store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")),
- memStoreSize);
+ memStoreSizing);
}
// flush them
flushStore(store, seqID);
for (int i = 41; i < 50; i++) {
store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")),
- memStoreSize);
+ memStoreSizing);
}
// flush them
flushStore(store, seqID);
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb506fd4/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
index 63c904d..d23dd1a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -77,8 +77,8 @@ import org.apache.hadoop.hbase.regionserver.FlushRequester;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.HStore;
+import org.apache.hadoop.hbase.regionserver.MemStoreSizing;
import org.apache.hadoop.hbase.regionserver.MemStoreSnapshot;
-import org.apache.hadoop.hbase.regionserver.MemStoreSize;
import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
@@ -543,8 +543,8 @@ public abstract class AbstractTestWALReplay {
final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
@Override
- protected void restoreEdit(HStore s, Cell cell, MemStoreSize memstoreSize) {
- super.restoreEdit(s, cell, memstoreSize);
+ protected void restoreEdit(HStore s, Cell cell, MemStoreSizing memstoreSizing) {
+ super.restoreEdit(s, cell, memstoreSizing);
countOfRestoredEdits.incrementAndGet();
}
};
[10/23] hbase git commit: HBASE-19073 Cleanup CoordinatedStateManager
Posted by zh...@apache.org.
HBASE-19073 Cleanup CoordinatedStateManager
- Merged BaseCSM class into CSM interface
- Removed config hbase.coordinated.state.manager.class
- Since state manager is not pluggable anymore, we don't need start/stop/initialize to setup unknown classes. Our internal ZkCSM now requires Server in constructor itself. Makes the dependency clearer too.
- Removed CSM from HRegionServer and HMaster constructor. Although it's a step back from dependency injection, but it's more consistent with our current (not good) pattern where we initialize everything in the ctor itself.
Change-Id: Ifca06bb354adec5b11ea1bad4707e014410491fc
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dd70cc30
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dd70cc30
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dd70cc30
Branch: refs/heads/HBASE-18410
Commit: dd70cc308158c435c6d8ec027e2435a29be4326b
Parents: eee3b01
Author: Apekshit Sharma <ap...@apache.org>
Authored: Fri Oct 20 17:20:17 2017 -0700
Committer: Apekshit Sharma <ap...@apache.org>
Committed: Tue Oct 24 19:56:04 2017 -0700
----------------------------------------------------------------------
.../master/LogRollMasterProcedureManager.java | 9 +--
.../LogRollRegionServerProcedureManager.java | 12 ++--
.../org/apache/hadoop/hbase/HConstants.java | 6 +-
.../hadoop/hbase/CoordinatedStateManager.java | 31 ++++----
.../hbase/CoordinatedStateManagerFactory.java | 48 -------------
.../apache/hadoop/hbase/LocalHBaseCluster.java | 16 +----
.../BaseCoordinatedStateManager.java | 76 --------------------
.../SplitLogManagerCoordination.java | 4 +-
.../ZKSplitLogManagerCoordination.java | 9 +--
.../coordination/ZkCoordinatedStateManager.java | 21 +++---
.../ZkSplitLogWorkerCoordination.java | 9 +--
.../org/apache/hadoop/hbase/master/HMaster.java | 12 ++--
.../hadoop/hbase/master/HMasterCommandLine.java | 10 +--
.../hadoop/hbase/master/SplitLogManager.java | 4 +-
.../hbase/regionserver/HRegionServer.java | 29 +++-----
.../regionserver/HRegionServerCommandLine.java | 5 +-
.../hbase/regionserver/SplitLogWorker.java | 15 ++--
.../hadoop/hbase/util/JVMClusterUtil.java | 23 ++----
.../apache/hadoop/hbase/wal/WALSplitter.java | 49 ++++++-------
.../apache/hadoop/hbase/MiniHBaseCluster.java | 4 +-
.../hadoop/hbase/TestLocalHBaseCluster.java | 10 ++-
.../hadoop/hbase/TestMovedRegionsCleaner.java | 5 +-
.../client/TestClientScannerRPCTimeout.java | 6 +-
.../hadoop/hbase/client/TestMetaCache.java | 4 +-
.../master/TestDistributedLogSplitting.java | 5 +-
.../hbase/master/TestHMasterRPCException.java | 5 +-
.../hadoop/hbase/master/TestMasterMetrics.java | 5 +-
.../hbase/master/TestMasterNoCluster.java | 17 ++---
.../hbase/master/TestMetaShutdownHandler.java | 5 +-
.../hbase/master/TestSplitLogManager.java | 6 +-
.../hbase/regionserver/OOMERegionServer.java | 5 +-
.../hbase/regionserver/TestClusterId.java | 10 +--
.../TestCompactionInDeadRegionServer.java | 4 --
.../hbase/regionserver/TestPriorityRpc.java | 5 +-
.../TestRSKilledWhenInitializing.java | 4 +-
.../TestRegionMergeTransactionOnCluster.java | 6 +-
.../regionserver/TestRegionServerHostname.java | 2 +-
.../TestRegionServerReportForDuty.java | 5 +-
.../TestScannerHeartbeatMessages.java | 5 --
.../hbase/regionserver/TestSplitLogWorker.java | 5 +-
.../TestSplitTransactionOnCluster.java | 6 +-
.../regionserver/wal/AbstractTestWALReplay.java | 6 +-
.../replication/TestReplicationSource.java | 5 --
.../hbase/wal/TestWALReaderOnSecureWAL.java | 10 +--
.../apache/hadoop/hbase/wal/TestWALSplit.java | 14 ++--
src/main/asciidoc/_chapters/hbase-default.adoc | 11 ---
46 files changed, 158 insertions(+), 405 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java
index 62b2df7..567d5ec 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java
@@ -26,12 +26,12 @@ import java.util.concurrent.ThreadPoolExecutor;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
import org.apache.hadoop.hbase.backup.impl.BackupManager;
+import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
+import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.errorhandling.ForeignException;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.master.MasterServices;
@@ -95,10 +95,7 @@ public class LogRollMasterProcedureManager extends MasterProcedureManager {
// setup the default procedure coordinator
ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads);
- BaseCoordinatedStateManager coordManager =
- (BaseCoordinatedStateManager) CoordinatedStateManagerFactory
- .getCoordinatedStateManager(master.getConfiguration());
- coordManager.initialize(master);
+ CoordinatedStateManager coordManager = new ZkCoordinatedStateManager(master);
ProcedureCoordinatorRpcs comms =
coordManager.getProcedureCoordinatorRpcs(getProcedureSignature(), name);
this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java
index 5ab7fac..c499ec7 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java
@@ -24,12 +24,12 @@ import java.util.concurrent.ThreadPoolExecutor;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
+import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
import org.apache.hadoop.hbase.backup.impl.BackupManager;
import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
+import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.procedure.ProcedureMember;
import org.apache.hadoop.hbase.procedure.ProcedureMemberRpcs;
@@ -159,12 +159,8 @@ public class LogRollRegionServerProcedureManager extends RegionServerProcedureMa
+ " setting");
return;
}
- BaseCoordinatedStateManager coordManager =
- (BaseCoordinatedStateManager) CoordinatedStateManagerFactory.
- getCoordinatedStateManager(rss.getConfiguration());
- coordManager.initialize(rss);
- this.memberRpcs =
- coordManager
+ CoordinatedStateManager coordManager = new ZkCoordinatedStateManager(rss);
+ this.memberRpcs = coordManager
.getProcedureMemberRpcs(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE);
// read in the backup handler configuration properties
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index a272fc8..45f22a6 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1209,10 +1209,6 @@ public final class HConstants {
public static final int REPLICATION_SOURCE_MAXTHREADS_DEFAULT = 10;
- /** Config for pluggable consensus provider */
- public static final String HBASE_COORDINATED_STATE_MANAGER_CLASS =
- "hbase.coordinated.state.manager.class";
-
/** Configuration key for SplitLog manager timeout */
public static final String HBASE_SPLITLOG_MANAGER_TIMEOUT = "hbase.splitlog.manager.timeout";
@@ -1298,7 +1294,7 @@ public final class HConstants {
public static final String HBASE_CANARY_WRITE_TABLE_CHECK_PERIOD_KEY =
"hbase.canary.write.table.check.period";
-
+
public static final String HBASE_CANARY_READ_RAW_SCAN_KEY = "hbase.canary.read.raw.enabled";
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/CoordinatedStateManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/CoordinatedStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/CoordinatedStateManager.java
index 1da7587..ab146e7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/CoordinatedStateManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/CoordinatedStateManager.java
@@ -17,7 +17,14 @@
*/
package org.apache.hadoop.hbase;
+import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination;
+import org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
+import org.apache.hadoop.hbase.procedure.ProcedureMemberRpcs;
import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.zookeeper.KeeperException;
+
+import java.io.IOException;
/**
* Implementations of this interface will keep and return to clients
@@ -28,31 +35,27 @@ import org.apache.yetus.audience.InterfaceAudience;
* For each coarse-grained area of operations there will be a separate
* interface with implementation, providing API for relevant operations
* requiring coordination.
- *
- * Property hbase.coordinated.state.manager.class in hbase-site.xml controls
- * which provider to use.
*/
@InterfaceAudience.Private
public interface CoordinatedStateManager {
-
/**
- * Initialize coordinated state management service.
- * @param server server instance to run within.
+ * Method to retrieve coordination for split log worker
*/
- void initialize(Server server);
+ SplitLogWorkerCoordination getSplitLogWorkerCoordination();
/**
- * Starts service.
+ * Method to retrieve coordination for split log manager
*/
- void start();
-
+ SplitLogManagerCoordination getSplitLogManagerCoordination();
/**
- * Stops service.
+ * Method to retrieve {@link org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs}
*/
- void stop();
+ ProcedureCoordinatorRpcs getProcedureCoordinatorRpcs(String procType, String coordNode)
+ throws IOException;
/**
- * @return instance of Server coordinated state manager runs within
+ * Method to retrieve {@link org.apache.hadoop.hbase.procedure.ProcedureMemberRpcs}
*/
- Server getServer();
+ ProcedureMemberRpcs getProcedureMemberRpcs(String procType) throws KeeperException;
+
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/CoordinatedStateManagerFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/CoordinatedStateManagerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/CoordinatedStateManagerFactory.java
deleted file mode 100644
index c66215a..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/CoordinatedStateManagerFactory.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase;
-
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
-import org.apache.hadoop.util.ReflectionUtils;
-
-/**
- * Creates instance of {@link CoordinatedStateManager}
- * based on configuration.
- */
-@InterfaceAudience.Private
-public final class CoordinatedStateManagerFactory {
-
- /**
- * Private to keep this class from being accidentally instantiated.
- */
- private CoordinatedStateManagerFactory(){}
-
- /**
- * Creates consensus provider from the given configuration.
- * @param conf Configuration
- * @return Implementation of {@link CoordinatedStateManager}
- */
- public static CoordinatedStateManager getCoordinatedStateManager(Configuration conf) {
- Class<? extends CoordinatedStateManager> coordinatedStateMgrKlass =
- conf.getClass(HConstants.HBASE_COORDINATED_STATE_MANAGER_CLASS,
- ZkCoordinatedStateManager.class, CoordinatedStateManager.class);
- return ReflectionUtils.newInstance(coordinatedStateMgrKlass, conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
index 123cc42..e43d33b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
@@ -173,14 +173,8 @@ public class LocalHBaseCluster {
// Create each regionserver with its own Configuration instance so each has
// its Connection instance rather than share (see HBASE_INSTANCES down in
// the guts of ConnectionManager).
-
- // Also, create separate CoordinatedStateManager instance per Server.
- // This is special case when we have to have more than 1 CoordinatedStateManager
- // within 1 process.
- CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf);
-
JVMClusterUtil.RegionServerThread rst =
- JVMClusterUtil.createRegionServerThread(config, cp, (Class<? extends HRegionServer>) conf
+ JVMClusterUtil.createRegionServerThread(config, (Class<? extends HRegionServer>) conf
.getClass(HConstants.REGION_SERVER_IMPL, this.regionServerClass), index);
this.regionThreads.add(rst);
@@ -208,13 +202,7 @@ public class LocalHBaseCluster {
// Create each master with its own Configuration instance so each has
// its Connection instance rather than share (see HBASE_INSTANCES down in
// the guts of ConnectionManager.
-
- // Also, create separate CoordinatedStateManager instance per Server.
- // This is special case when we have to have more than 1 CoordinatedStateManager
- // within 1 process.
- CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf);
-
- JVMClusterUtil.MasterThread mt = JVMClusterUtil.createMasterThread(c, cp,
+ JVMClusterUtil.MasterThread mt = JVMClusterUtil.createMasterThread(c,
(Class<? extends HMaster>) conf.getClass(HConstants.MASTER_IMPL, this.masterClass), index);
this.masterThreads.add(mt);
return mt;
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/BaseCoordinatedStateManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/BaseCoordinatedStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/BaseCoordinatedStateManager.java
deleted file mode 100644
index 0727375..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/BaseCoordinatedStateManager.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.coordination;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.CoordinatedStateManager;
-import org.apache.hadoop.hbase.Server;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
-import org.apache.hadoop.hbase.procedure.ProcedureMemberRpcs;
-import org.apache.zookeeper.KeeperException;
-
-/**
- * Base class for {@link org.apache.hadoop.hbase.CoordinatedStateManager} implementations.
- * Defines methods to retrieve coordination objects for relevant areas. CoordinatedStateManager
- * reference returned from Server interface has to be casted to this type to
- * access those methods.
- */
-@InterfaceAudience.Private
-public abstract class BaseCoordinatedStateManager implements CoordinatedStateManager {
-
- @Override
- public void initialize(Server server) {
- }
-
- @Override
- public void start() {
- }
-
- @Override
- public void stop() {
- }
-
- @Override
- public Server getServer() {
- return null;
- }
-
- /**
- * Method to retrieve coordination for split log worker
- */
- public abstract SplitLogWorkerCoordination getSplitLogWorkerCoordination();
-
- /**
- * Method to retrieve coordination for split log manager
- */
- public abstract SplitLogManagerCoordination getSplitLogManagerCoordination();
- /**
- * Method to retrieve {@link org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs}
- */
- public abstract ProcedureCoordinatorRpcs
- getProcedureCoordinatorRpcs(String procType, String coordNode) throws IOException;
-
- /**
- * Method to retrieve {@link org.apache.hadoop.hbase.procedure.ProcedureMemberRpcs}
- */
- public abstract ProcedureMemberRpcs
- getProcedureMemberRpcs(String procType) throws KeeperException;
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
index df8103b..564d729 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
@@ -58,7 +58,7 @@ public interface SplitLogManagerCoordination {
/**
* Detail class that shares data between coordination and split log manager
*/
- public static class SplitLogManagerDetails {
+ class SplitLogManagerDetails {
final private ConcurrentMap<String, Task> tasks;
final private MasterServices master;
final private Set<String> failedDeletions;
@@ -156,7 +156,7 @@ public interface SplitLogManagerCoordination {
* @throws InterruptedIOException
* @throws IOException in case of failure
*/
- void setRecoveryMode(boolean b) throws InterruptedIOException, IOException;
+ void setRecoveryMode(boolean b) throws IOException;
/**
* Removes known stale servers
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java
index 5fd20e8..ef99de0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.master.SplitLogManager.ResubmitDirective;
import org.apache.hadoop.hbase.master.SplitLogManager.Task;
import org.apache.hadoop.hbase.master.SplitLogManager.TerminationStatus;
+import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.hadoop.hbase.wal.WALSplitter;
@@ -101,14 +102,14 @@ public class ZKSplitLogManagerCoordination extends ZooKeeperListener implements
private boolean isDrainingDone = false;
- public ZKSplitLogManagerCoordination(final CoordinatedStateManager manager,
- ZooKeeperWatcher watcher) {
+ public ZKSplitLogManagerCoordination(Configuration conf, ZooKeeperWatcher watcher) {
super(watcher);
+ this.conf = conf;
taskFinisher = new TaskFinisher() {
@Override
public Status finish(ServerName workerName, String logfile) {
try {
- WALSplitter.finishSplitLogFile(logfile, manager.getServer().getConfiguration());
+ WALSplitter.finishSplitLogFile(logfile, conf);
} catch (IOException e) {
LOG.warn("Could not finish splitting of log file " + logfile, e);
return Status.ERR;
@@ -116,7 +117,6 @@ public class ZKSplitLogManagerCoordination extends ZooKeeperListener implements
return Status.DONE;
}
};
- this.conf = manager.getServer().getConfiguration();
}
@Override
@@ -1122,6 +1122,7 @@ public class ZKSplitLogManagerCoordination extends ZooKeeperListener implements
/**
* Temporary function that is used by unit tests only
*/
+ @VisibleForTesting
public void setIgnoreDeleteForTesting(boolean b) {
ignoreZKDeleteForTesting = b;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkCoordinatedStateManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkCoordinatedStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkCoordinatedStateManager.java
index 2fc8e39..10e2642 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkCoordinatedStateManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkCoordinatedStateManager.java
@@ -19,8 +19,11 @@ package org.apache.hadoop.hbase.coordination;
import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
import org.apache.hadoop.hbase.procedure.ProcedureMemberRpcs;
@@ -33,24 +36,16 @@ import org.apache.zookeeper.KeeperException;
* ZooKeeper-based implementation of {@link org.apache.hadoop.hbase.CoordinatedStateManager}.
*/
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
-public class ZkCoordinatedStateManager extends BaseCoordinatedStateManager {
- protected Server server;
+public class ZkCoordinatedStateManager implements CoordinatedStateManager {
protected ZooKeeperWatcher watcher;
protected SplitLogWorkerCoordination splitLogWorkerCoordination;
protected SplitLogManagerCoordination splitLogManagerCoordination;
- @Override
- public void initialize(Server server) {
- this.server = server;
+ public ZkCoordinatedStateManager(Server server) {
this.watcher = server.getZooKeeper();
- splitLogWorkerCoordination = new ZkSplitLogWorkerCoordination(this, watcher);
- splitLogManagerCoordination = new ZKSplitLogManagerCoordination(this, watcher);
-
- }
-
- @Override
- public Server getServer() {
- return server;
+ splitLogWorkerCoordination = new ZkSplitLogWorkerCoordination(server.getServerName(), watcher);
+ splitLogManagerCoordination = new ZKSplitLogManagerCoordination(server.getConfiguration(),
+ watcher);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java
index 230a42f..14e7796 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java
@@ -90,13 +90,11 @@ public class ZkSplitLogWorkerCoordination extends ZooKeeperListener implements
protected final AtomicInteger tasksInProgress = new AtomicInteger(0);
private int maxConcurrentTasks = 0;
- private final ZkCoordinatedStateManager manager;
+ private final ServerName serverName;
- public ZkSplitLogWorkerCoordination(ZkCoordinatedStateManager zkCoordinatedStateManager,
- ZooKeeperWatcher watcher) {
+ public ZkSplitLogWorkerCoordination(ServerName serverName, ZooKeeperWatcher watcher) {
super(watcher);
- manager = zkCoordinatedStateManager;
-
+ this.serverName = serverName;
}
/**
@@ -185,7 +183,6 @@ public class ZkSplitLogWorkerCoordination extends ZooKeeperListener implements
// currentTask can change but that's ok
String taskpath = currentTask;
if (taskpath != null && taskpath.equals(path)) {
- ServerName serverName = manager.getServer().getServerName();
// have to compare data. cannot compare version because then there
// will be race with attemptToOwnTask()
// cannot just check whether the node has been transitioned to
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index bb36520..a990a4b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -468,13 +468,12 @@ public class HMaster extends HRegionServer implements MasterServices {
* #finishActiveMasterInitialization(MonitoredTask) after
* the master becomes the active one.
*/
- public HMaster(final Configuration conf, CoordinatedStateManager csm)
+ public HMaster(final Configuration conf)
throws IOException, KeeperException {
- super(conf, csm);
+ super(conf);
try {
this.rsFatals = new MemoryBoundedLogMessageBuffer(
conf.getLong("hbase.master.buffer.for.rs.fatals", 1 * 1024 * 1024));
-
LOG.info("hbase.rootdir=" + getRootDir() +
", hbase.cluster.distributed=" + this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false));
@@ -2828,11 +2827,10 @@ public class HMaster extends HRegionServer implements MasterServices {
* @return HMaster instance.
*/
public static HMaster constructMaster(Class<? extends HMaster> masterClass,
- final Configuration conf, final CoordinatedStateManager cp) {
+ final Configuration conf) {
try {
- Constructor<? extends HMaster> c =
- masterClass.getConstructor(Configuration.class, CoordinatedStateManager.class);
- return c.newInstance(conf, cp);
+ Constructor<? extends HMaster> c = masterClass.getConstructor(Configuration.class);
+ return c.newInstance(conf);
} catch(Exception e) {
Throwable error = e;
if (e instanceof InvocationTargetException &&
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
index d1e6b73..f9a441d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
@@ -29,8 +29,6 @@ import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.CoordinatedStateManager;
-import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.LocalHBaseCluster;
import org.apache.hadoop.hbase.MasterNotRunningException;
@@ -230,9 +228,7 @@ public class HMasterCommandLine extends ServerCommandLine {
waitOnMasterThreads(cluster);
} else {
logProcessInfo(getConf());
- CoordinatedStateManager csm =
- CoordinatedStateManagerFactory.getCoordinatedStateManager(conf);
- HMaster master = HMaster.constructMaster(masterClass, conf, csm);
+ HMaster master = HMaster.constructMaster(masterClass, conf);
if (master.isStopped()) {
LOG.info("Won't bring the Master up as a shutdown is requested");
return 1;
@@ -302,9 +298,9 @@ public class HMasterCommandLine extends ServerCommandLine {
public static class LocalHMaster extends HMaster {
private MiniZooKeeperCluster zkcluster = null;
- public LocalHMaster(Configuration conf, CoordinatedStateManager csm)
+ public LocalHMaster(Configuration conf)
throws IOException, KeeperException, InterruptedException {
- super(conf, csm);
+ super(conf);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index 8027b6a..67aab82 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.SplitLogCounters;
import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.client.RegionInfo;
-import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination;
import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLogManagerDetails;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
@@ -153,8 +152,7 @@ public class SplitLogManager {
}
private SplitLogManagerCoordination getSplitLogManagerCoordination() {
- return ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
- .getSplitLogManagerCoordination();
+ return server.getCoordinatedStateManager().getSplitLogManagerCoordination();
}
private FileStatus[] getFileList(List<Path> logDirs, PathFilter filter) throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index d792628..f384c1f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -64,7 +64,6 @@ import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.ChoreService;
import org.apache.hadoop.hbase.ClockOutOfSyncException;
import org.apache.hadoop.hbase.CoordinatedStateManager;
-import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
@@ -92,8 +91,8 @@ import org.apache.hadoop.hbase.client.locking.EntityLock;
import org.apache.hadoop.hbase.client.locking.LockServiceClient;
import org.apache.hadoop.hbase.conf.ConfigurationManager;
import org.apache.hadoop.hbase.conf.ConfigurationObserver;
-import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
import org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination;
+import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.exceptions.RegionMovedException;
import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
@@ -505,7 +504,7 @@ public class HRegionServer extends HasThread implements
protected final RSRpcServices rpcServices;
- protected BaseCoordinatedStateManager csm;
+ protected CoordinatedStateManager csm;
/**
* Configuration manager is used to register/deregister and notify the configuration observers
@@ -533,21 +532,13 @@ public class HRegionServer extends HasThread implements
private final boolean masterless;
static final String MASTERLESS_CONFIG_NAME = "hbase.masterless";
- /**
- * Starts a HRegionServer at the default location.
- */
- public HRegionServer(Configuration conf) throws IOException, InterruptedException {
- this(conf, CoordinatedStateManagerFactory.getCoordinatedStateManager(conf));
- }
/**
* Starts a HRegionServer at the default location
- *
- * @param csm implementation of CoordinatedStateManager to be used
*/
- // Don't start any services or managers in here in the Consructor.
+ // Don't start any services or managers in here in the Constructor.
// Defer till after we register with the Master as much as possible. See #startServices.
- public HRegionServer(Configuration conf, CoordinatedStateManager csm) throws IOException {
+ public HRegionServer(Configuration conf) throws IOException {
super("RegionServer"); // thread name
try {
this.startcode = System.currentTimeMillis();
@@ -642,9 +633,7 @@ public class HRegionServer extends HasThread implements
// If no master in cluster, skip trying to track one or look for a cluster status.
if (!this.masterless) {
- this.csm = (BaseCoordinatedStateManager) csm;
- this.csm.initialize(this);
- this.csm.start();
+ this.csm = new ZkCoordinatedStateManager(this);
masterAddressTracker = new MasterAddressTracker(getZooKeeper(), this);
masterAddressTracker.start();
@@ -2924,7 +2913,7 @@ public class HRegionServer extends HasThread implements
}
@Override
- public BaseCoordinatedStateManager getCoordinatedStateManager() {
+ public CoordinatedStateManager getCoordinatedStateManager() {
return csm;
}
@@ -3020,11 +3009,11 @@ public class HRegionServer extends HasThread implements
*/
public static HRegionServer constructRegionServer(
Class<? extends HRegionServer> regionServerClass,
- final Configuration conf2, CoordinatedStateManager cp) {
+ final Configuration conf2) {
try {
Constructor<? extends HRegionServer> c = regionServerClass
- .getConstructor(Configuration.class, CoordinatedStateManager.class);
- return c.newInstance(conf2, cp);
+ .getConstructor(Configuration.class);
+ return c.newInstance(conf2);
} catch (Exception e) {
throw new RuntimeException("Failed construction of " + "Regionserver: "
+ regionServerClass.toString(), e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java
index 343ebf6..1212668 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java
@@ -23,10 +23,8 @@ import org.apache.commons.logging.LogFactory;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.LocalHBaseCluster;
-import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.util.ServerCommandLine;
/**
@@ -52,7 +50,6 @@ public class HRegionServerCommandLine extends ServerCommandLine {
private int start() throws Exception {
Configuration conf = getConf();
- CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf);
try {
// If 'local', don't start a region server here. Defer to
// LocalHBaseCluster. It manages 'local' clusters.
@@ -61,7 +58,7 @@ public class HRegionServerCommandLine extends ServerCommandLine {
+ HConstants.CLUSTER_DISTRIBUTED + " is false");
} else {
logProcessInfo(getConf());
- HRegionServer hrs = HRegionServer.constructRegionServer(regionServerClass, conf, cp);
+ HRegionServer hrs = HRegionServer.constructRegionServer(regionServerClass, conf);
hrs.start();
hrs.join();
if (hrs.isAborted()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
index 96e9cf5..9d00f1a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.client.RetriesExhaustedException;
-import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
import org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
import org.apache.hadoop.hbase.wal.WALFactory;
@@ -45,8 +44,8 @@ import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTe
/**
* This worker is spawned in every regionserver, including master. The Worker waits for log
- * splitting tasks to be put up by the {@link org.apache.hadoop.hbase.master.SplitLogManager}
- * running in the master and races with other workers in other serves to acquire those tasks.
+ * splitting tasks to be put up by the {@link org.apache.hadoop.hbase.master.SplitLogManager}
+ * running in the master and races with other workers in other serves to acquire those tasks.
* The coordination is done via coordination engine.
* <p>
* If a worker has successfully moved the task from state UNASSIGNED to OWNED then it owns the task.
@@ -75,9 +74,7 @@ public class SplitLogWorker implements Runnable {
TaskExecutor splitTaskExecutor) {
this.server = server;
this.conf = conf;
- this.coordination =
- ((BaseCoordinatedStateManager) hserver.getCoordinatedStateManager())
- .getSplitLogWorkerCoordination();
+ this.coordination = hserver.getCoordinatedStateManager().getSplitLogWorkerCoordination();
this.server = server;
coordination.init(server, conf, splitTaskExecutor, this);
}
@@ -102,7 +99,9 @@ public class SplitLogWorker implements Runnable {
// encountered a bad non-retry-able persistent error.
try {
if (!WALSplitter.splitLogFile(walDir, fs.getFileStatus(new Path(walDir, filename)),
- fs, conf, p, sequenceIdChecker, server.getCoordinatedStateManager(), mode, factory)) {
+ fs, conf, p, sequenceIdChecker,
+ server.getCoordinatedStateManager().getSplitLogWorkerCoordination(),
+ server.getConnection(), mode, factory)) {
return Status.PREEMPTED;
}
} catch (InterruptedIOException iioe) {
@@ -186,7 +185,7 @@ public class SplitLogWorker implements Runnable {
* acquired by a {@link SplitLogWorker}. Since there isn't a water-tight
* guarantee that two workers will not be executing the same task therefore it
* is better to have workers prepare the task and then have the
- * {@link org.apache.hadoop.hbase.master.SplitLogManager} commit the work in
+ * {@link org.apache.hadoop.hbase.master.SplitLogManager} commit the work in
* SplitLogManager.TaskFinisher
*/
public interface TaskExecutor {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
index 135dabb..b88c0e6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
@@ -72,23 +72,18 @@ public class JVMClusterUtil {
* Creates a {@link RegionServerThread}.
* Call 'start' on the returned thread to make it run.
* @param c Configuration to use.
- * @param cp consensus provider to use
* @param hrsc Class to create.
* @param index Used distinguishing the object returned.
* @throws IOException
* @return Region server added.
*/
- public static JVMClusterUtil.RegionServerThread createRegionServerThread(
- final Configuration c, CoordinatedStateManager cp, final Class<? extends HRegionServer> hrsc,
- final int index)
- throws IOException {
+ public static JVMClusterUtil.RegionServerThread createRegionServerThread(final Configuration c,
+ final Class<? extends HRegionServer> hrsc, final int index) throws IOException {
HRegionServer server;
try {
-
- Constructor<? extends HRegionServer> ctor = hrsc.getConstructor(Configuration.class,
- CoordinatedStateManager.class);
+ Constructor<? extends HRegionServer> ctor = hrsc.getConstructor(Configuration.class);
ctor.setAccessible(true);
- server = ctor.newInstance(c, cp);
+ server = ctor.newInstance(c);
} catch (InvocationTargetException ite) {
Throwable target = ite.getTargetException();
throw new RuntimeException("Failed construction of RegionServer: " +
@@ -124,20 +119,16 @@ public class JVMClusterUtil {
* Creates a {@link MasterThread}.
* Call 'start' on the returned thread to make it run.
* @param c Configuration to use.
- * @param cp consensus provider to use
* @param hmc Class to create.
* @param index Used distinguishing the object returned.
* @throws IOException
* @return Master added.
*/
- public static JVMClusterUtil.MasterThread createMasterThread(
- final Configuration c, CoordinatedStateManager cp, final Class<? extends HMaster> hmc,
- final int index)
- throws IOException {
+ public static JVMClusterUtil.MasterThread createMasterThread(final Configuration c,
+ final Class<? extends HMaster> hmc, final int index) throws IOException {
HMaster server;
try {
- server = hmc.getConstructor(Configuration.class, CoordinatedStateManager.class).
- newInstance(c, cp);
+ server = hmc.getConstructor(Configuration.class).newInstance(c);
} catch (InvocationTargetException ite) {
Throwable target = ite.getTargetException();
throw new RuntimeException("Failed construction of Master: " +
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
index aeacd9d..e39d7ab 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
@@ -18,6 +18,9 @@
*/
package org.apache.hadoop.hbase.wal;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination;
+import org.apache.hadoop.hbase.regionserver.SplitLogWorker;
import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hbase.shaded.com.google.common.base.Preconditions;
import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
@@ -82,7 +85,6 @@ import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.TableState;
-import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
import org.apache.hadoop.hbase.coordination.ZKSplitLogManagerCoordination;
import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
import org.apache.hadoop.hbase.io.HeapSize;
@@ -146,7 +148,8 @@ public class WALSplitter {
private Map<TableName, TableState> tableStatesCache =
new ConcurrentHashMap<>();
- private BaseCoordinatedStateManager csm;
+ private SplitLogWorkerCoordination splitLogWorkerCoordination;
+ private Connection connection;
private final WALFactory walFactory;
private MonitoredTask status;
@@ -177,7 +180,8 @@ public class WALSplitter {
@VisibleForTesting
WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,
FileSystem fs, LastSequenceId idChecker,
- CoordinatedStateManager csm, RecoveryMode mode) {
+ SplitLogWorkerCoordination splitLogWorkerCoordination, Connection connection,
+ RecoveryMode mode) {
this.conf = HBaseConfiguration.create(conf);
String codecClassName = conf
.get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());
@@ -185,7 +189,9 @@ public class WALSplitter {
this.rootDir = rootDir;
this.fs = fs;
this.sequenceIdChecker = idChecker;
- this.csm = (BaseCoordinatedStateManager)csm;
+ this.splitLogWorkerCoordination = splitLogWorkerCoordination;
+ this.connection = connection;
+
this.walFactory = factory;
this.controller = new PipelineController();
@@ -199,7 +205,7 @@ public class WALSplitter {
this.distributedLogReplay = (RecoveryMode.LOG_REPLAY == mode);
this.numWriterThreads = this.conf.getInt("hbase.regionserver.hlog.splitlog.writer.threads", 3);
- if (csm != null && this.distributedLogReplay) {
+ if (this.splitLogWorkerCoordination != null && this.distributedLogReplay) {
outputSink = new LogReplayOutputSink(controller, entryBuffers, numWriterThreads);
} else {
if (this.distributedLogReplay) {
@@ -217,20 +223,14 @@ public class WALSplitter {
* <p>
* If the log file has N regions then N recovered.edits files will be produced.
* <p>
- * @param rootDir
- * @param logfile
- * @param fs
- * @param conf
- * @param reporter
- * @param idChecker
- * @param cp coordination state manager
* @return false if it is interrupted by the progress-able.
- * @throws IOException
*/
public static boolean splitLogFile(Path rootDir, FileStatus logfile, FileSystem fs,
Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker,
- CoordinatedStateManager cp, RecoveryMode mode, final WALFactory factory) throws IOException {
- WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker, cp, mode);
+ SplitLogWorkerCoordination splitLogWorkerCoordination, Connection connection,
+ RecoveryMode mode, final WALFactory factory) throws IOException {
+ WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker,
+ splitLogWorkerCoordination, connection, mode);
return s.splitLogFile(logfile, reporter);
}
@@ -246,7 +246,7 @@ public class WALSplitter {
List<Path> splits = new ArrayList<>();
if (logfiles != null && logfiles.length > 0) {
for (FileStatus logfile: logfiles) {
- WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, null, null,
+ WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, null, null, null,
RecoveryMode.LOG_SPLITTING);
if (s.splitLogFile(logfile, null)) {
finishSplitLogFile(rootDir, oldLogDir, logfile.getPath(), conf);
@@ -317,9 +317,8 @@ public class WALSplitter {
lastFlushedSequenceId = lastFlushedSequenceIds.get(encodedRegionNameAsStr);
if (lastFlushedSequenceId == null) {
if (this.distributedLogReplay) {
- RegionStoreSequenceIds ids =
- csm.getSplitLogWorkerCoordination().getRegionFlushedSequenceId(failedServerName,
- encodedRegionNameAsStr);
+ RegionStoreSequenceIds ids = splitLogWorkerCoordination.getRegionFlushedSequenceId(
+ failedServerName, encodedRegionNameAsStr);
if (ids != null) {
lastFlushedSequenceId = ids.getLastFlushedSequenceId();
if (LOG.isDebugEnabled()) {
@@ -377,10 +376,9 @@ public class WALSplitter {
throw iie;
} catch (CorruptedLogFileException e) {
LOG.warn("Could not parse, corrupted WAL=" + logPath, e);
- if (this.csm != null) {
+ if (splitLogWorkerCoordination != null) {
// Some tests pass in a csm of null.
- this.csm.getSplitLogWorkerCoordination().markCorrupted(rootDir,
- logfile.getPath().getName(), fs);
+ splitLogWorkerCoordination.markCorrupted(rootDir, logfile.getPath().getName(), fs);
} else {
// for tests only
ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);
@@ -1952,7 +1950,7 @@ public class WALSplitter {
// retrieve last flushed sequence Id from ZK. Because region postOpenDeployTasks will
// update the value for the region
RegionStoreSequenceIds ids =
- csm.getSplitLogWorkerCoordination().getRegionFlushedSequenceId(failedServerName,
+ splitLogWorkerCoordination.getRegionFlushedSequenceId(failedServerName,
loc.getRegionInfo().getEncodedName());
if (ids != null) {
lastFlushedSequenceId = ids.getLastFlushedSequenceId();
@@ -2185,15 +2183,14 @@ public class WALSplitter {
}
private boolean isTableDisabledOrDisabling(TableName tableName) {
- if (csm == null)
+ if (connection == null)
return false; // we can't get state without CoordinatedStateManager
if (tableName.isSystemTable())
return false; // assume that system tables never can be disabled
TableState tableState = tableStatesCache.get(tableName);
if (tableState == null) {
try {
- tableState =
- MetaTableAccessor.getTableState(csm.getServer().getConnection(), tableName);
+ tableState = MetaTableAccessor.getTableState(connection, tableName);
if (tableState != null)
tableStatesCache.put(tableName, tableState);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
index fe4119a..a46fa9d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
@@ -109,9 +109,9 @@ public class MiniHBaseCluster extends HBaseCluster {
private Thread shutdownThread = null;
private User user = null;
- public MiniHBaseClusterRegionServer(Configuration conf, CoordinatedStateManager cp)
+ public MiniHBaseClusterRegionServer(Configuration conf)
throws IOException, InterruptedException {
- super(conf, cp);
+ super(conf);
this.user = User.getCurrent();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java
index bbf4f32..4c6b848 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestLocalHBaseCluster.java
@@ -66,10 +66,9 @@ public class TestLocalHBaseCluster {
* running in local mode.
*/
public static class MyHMaster extends HMaster {
- public MyHMaster(Configuration conf, CoordinatedStateManager cp)
- throws IOException, KeeperException,
+ public MyHMaster(Configuration conf) throws IOException, KeeperException,
InterruptedException {
- super(conf, cp);
+ super(conf);
}
public int echo(int val) {
@@ -82,9 +81,8 @@ public class TestLocalHBaseCluster {
*/
public static class MyHRegionServer extends MiniHBaseCluster.MiniHBaseClusterRegionServer {
- public MyHRegionServer(Configuration conf, CoordinatedStateManager cp) throws IOException,
- InterruptedException {
- super(conf, cp);
+ public MyHRegionServer(Configuration conf) throws IOException, InterruptedException {
+ super(conf);
}
public int echo(int val) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMovedRegionsCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMovedRegionsCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMovedRegionsCleaner.java
index 8d3d2fc..5fb16c7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMovedRegionsCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMovedRegionsCleaner.java
@@ -44,9 +44,8 @@ import java.io.IOException;
private static class TestMockRegionServer extends MiniHBaseCluster.MiniHBaseClusterRegionServer {
- public TestMockRegionServer(Configuration conf, CoordinatedStateManager cp)
- throws IOException, InterruptedException {
- super(conf, cp);
+ public TestMockRegionServer(Configuration conf) throws IOException, InterruptedException {
+ super(conf);
}
protected int movedRegionCleanerPeriod() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientScannerRPCTimeout.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientScannerRPCTimeout.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientScannerRPCTimeout.java
index de21f83..c9686c2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientScannerRPCTimeout.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientScannerRPCTimeout.java
@@ -141,9 +141,9 @@ public class TestClientScannerRPCTimeout {
}
private static class RegionServerWithScanTimeout extends MiniHBaseClusterRegionServer {
- public RegionServerWithScanTimeout(Configuration conf, CoordinatedStateManager cp)
+ public RegionServerWithScanTimeout(Configuration conf)
throws IOException, InterruptedException {
- super(conf, cp);
+ super(conf);
}
protected RSRpcServices createRpcServices() throws IOException {
@@ -168,7 +168,7 @@ public class TestClientScannerRPCTimeout {
throws ServiceException {
if (request.hasScannerId()) {
ScanResponse scanResponse = super.scan(controller, request);
- if (this.tableScannerId == request.getScannerId() &&
+ if (this.tableScannerId == request.getScannerId() &&
(sleepAlways || (!slept && seqNoToSleepOn == request.getNextCallSeq()))) {
try {
LOG.info("SLEEPING " + (rpcTimeout + 500));
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java
index e7aa60f..b1fda41 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java
@@ -198,9 +198,9 @@ public class TestMetaCache {
public static class RegionServerWithFakeRpcServices extends HRegionServer {
private FakeRSRpcServices rsRpcServices;
- public RegionServerWithFakeRpcServices(Configuration conf, CoordinatedStateManager cp)
+ public RegionServerWithFakeRpcServices(Configuration conf)
throws IOException, InterruptedException {
- super(conf, cp);
+ super(conf);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index a65924d..eafc412 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -82,7 +82,6 @@ import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
import org.apache.hadoop.hbase.coordination.ZKSplitLogManagerCoordination;
import org.apache.hadoop.hbase.exceptions.RegionInRecoveryException;
import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
@@ -1151,8 +1150,8 @@ public class TestDistributedLogSplitting {
out.write(Bytes.toBytes("corrupted bytes"));
out.close();
ZKSplitLogManagerCoordination coordination =
- (ZKSplitLogManagerCoordination) ((BaseCoordinatedStateManager) master
- .getCoordinatedStateManager()).getSplitLogManagerCoordination();
+ (ZKSplitLogManagerCoordination) (master.getCoordinatedStateManager())
+ .getSplitLogManagerCoordination();
coordination.setIgnoreDeleteForTesting(true);
executor = Executors.newSingleThreadExecutor();
Runnable runnable = new Runnable() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
index 8cb3ec8..64d5a02 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java
@@ -26,8 +26,6 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.CoordinatedStateManager;
-import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
@@ -68,10 +66,9 @@ public class TestHMasterRPCException {
conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 2000);
testUtil.startMiniZKCluster();
- CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf);
ZooKeeperWatcher watcher = testUtil.getZooKeeperWatcher();
ZKUtil.createWithParents(watcher, watcher.znodePaths.masterAddressZNode, Bytes.toBytes("fake:123"));
- master = new HMaster(conf, cp);
+ master = new HMaster(conf);
rpcClient = RpcClientFactory.createClient(conf, HConstants.CLUSTER_ID_DEFAULT);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
index 04cfc65..69baa5f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java
@@ -51,9 +51,8 @@ public class TestMasterMetrics {
private static HBaseTestingUtility TEST_UTIL;
public static class MyMaster extends HMaster {
- public MyMaster(Configuration conf, CoordinatedStateManager cp) throws IOException,
- KeeperException, InterruptedException {
- super(conf, cp);
+ public MyMaster(Configuration conf) throws IOException, KeeperException, InterruptedException {
+ super(conf);
}
/*
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
index 480ba9a..25671fc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
@@ -33,8 +33,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.CategoryBasedTimeout;
import org.apache.hadoop.hbase.CoordinatedStateException;
-import org.apache.hadoop.hbase.CoordinatedStateManager;
-import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -47,7 +45,6 @@ import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
-import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -133,9 +130,7 @@ public class TestMasterNoCluster {
@Test
public void testStopDuringStart()
throws IOException, KeeperException, InterruptedException {
- CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(
- TESTUTIL.getConfiguration());
- HMaster master = new HMaster(TESTUTIL.getConfiguration(), cp);
+ HMaster master = new HMaster(TESTUTIL.getConfiguration());
master.start();
// Immediately have it stop. We used hang in assigning meta.
master.stopMaster();
@@ -148,7 +143,7 @@ public class TestMasterNoCluster {
* @throws IOException
* @throws KeeperException
* @throws InterruptedException
- * @throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException
+ * @throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException
*/
@Ignore @Test // Disabled since HBASE-18511. Reenable when master can carry regions.
public void testFailover() throws Exception {
@@ -186,15 +181,13 @@ public class TestMasterNoCluster {
// and get notification on transitions. We need to fake out any rpcs the
// master does opening/closing regions. Also need to fake out the address
// of the 'remote' mocked up regionservers.
- CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(
- TESTUTIL.getConfiguration());
// Insert a mock for the connection, use TESTUTIL.getConfiguration rather than
// the conf from the master; the conf will already have an ClusterConnection
// associate so the below mocking of a connection will fail.
final ClusterConnection mockedConnection = HConnectionTestingUtility.getMockedConnectionAndDecorate(
TESTUTIL.getConfiguration(), rs0, rs0, rs0.getServerName(),
HRegionInfo.FIRST_META_REGIONINFO);
- HMaster master = new HMaster(conf, cp) {
+ HMaster master = new HMaster(conf) {
InetAddress getRemoteInetAddress(final int port, final long serverStartCode)
throws UnknownHostException {
// Return different address dependent on port passed.
@@ -262,9 +255,7 @@ public class TestMasterNoCluster {
final ServerName deadServer = ServerName.valueOf("test.sample", 1, 100);
final MockRegionServer rs0 = new MockRegionServer(conf, newServer);
- CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(
- TESTUTIL.getConfiguration());
- HMaster master = new HMaster(conf, cp) {
+ HMaster master = new HMaster(conf) {
@Override
MasterMetaBootstrap createMetaBootstrap(final HMaster master, final MonitoredTask status) {
return new MasterMetaBootstrap(this, status) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java
index 68160df..d50e4a3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java
@@ -124,10 +124,9 @@ public class TestMetaShutdownHandler {
public static class MyRegionServer extends MiniHBaseClusterRegionServer {
- public MyRegionServer(Configuration conf, CoordinatedStateManager cp)
- throws IOException, KeeperException,
+ public MyRegionServer(Configuration conf) throws IOException, KeeperException,
InterruptedException {
- super(conf, cp);
+ super(conf);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
index 04fc797..6215790 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
@@ -39,7 +39,6 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.UUID;
-import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.LongAdder;
import org.apache.commons.logging.Log;
@@ -48,7 +47,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CoordinatedStateManager;
-import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -57,6 +55,7 @@ import org.apache.hadoop.hbase.SplitLogCounters;
import org.apache.hadoop.hbase.SplitLogTask;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.coordination.ZKSplitLogManagerCoordination;
+import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
import org.apache.hadoop.hbase.master.SplitLogManager.Task;
import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch;
import org.apache.hadoop.hbase.regionserver.TestMasterAddressTracker.NodeCreationListener;
@@ -105,8 +104,7 @@ public class TestSplitLogManager {
public DummyMasterServices(ZooKeeperWatcher zkw, Configuration conf) {
super(conf);
this.zkw = zkw;
- cm = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf);
- cm.initialize(this);
+ cm = new ZkCoordinatedStateManager(this);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java
index 07c141c..ce38ef6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java
@@ -39,9 +39,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutateRequ
public class OOMERegionServer extends HRegionServer {
private List<Put> retainer = new ArrayList<>();
- public OOMERegionServer(HBaseConfiguration conf, CoordinatedStateManager cp)
- throws IOException, InterruptedException {
- super(conf, cp);
+ public OOMERegionServer(HBaseConfiguration conf) throws IOException, InterruptedException {
+ super(conf);
}
public void put(byte [] regionName, Put put)
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClusterId.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClusterId.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClusterId.java
index 2240226..46fd702 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClusterId.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClusterId.java
@@ -27,8 +27,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.CoordinatedStateManager;
-import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.master.LoadBalancer;
@@ -75,11 +73,9 @@ public class TestClusterId {
TEST_UTIL.startMiniDFSCluster(1);
Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
- CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf);
//start region server, needs to be separate
//so we get an unset clusterId
- rst = JVMClusterUtil.createRegionServerThread(conf,cp,
- HRegionServer.class, 0);
+ rst = JVMClusterUtil.createRegionServerThread(conf, HRegionServer.class, 0);
rst.start();
//Make sure RS is in blocking state
Thread.sleep(10000);
@@ -92,7 +88,7 @@ public class TestClusterId {
assertNotNull(clusterId);
assertEquals(clusterId, rst.getRegionServer().getClusterId());
}
-
+
@Test
public void testRewritingClusterIdToPB() throws Exception {
TEST_UTIL.startMiniZKCluster();
@@ -115,6 +111,6 @@ public class TestClusterId {
int expected = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration())? 2: 1;
assertEquals(expected, master.getServerManager().getOnlineServersList().size());
}
-
+
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionInDeadRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionInDeadRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionInDeadRegionServer.java
index 4b13c30..5e212f8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionInDeadRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionInDeadRegionServer.java
@@ -75,10 +75,6 @@ public class TestCompactionInDeadRegionServer {
super(conf);
}
- public IgnoreYouAreDeadRS(Configuration conf, CoordinatedStateManager csm) throws IOException {
- super(conf, csm);
- }
-
@Override
protected void tryRegionServerReport(long reportStartTime, long reportEndTime)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPriorityRpc.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPriorityRpc.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPriorityRpc.java
index ded4c48..ef216f7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPriorityRpc.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPriorityRpc.java
@@ -31,11 +31,9 @@ import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.ipc.PriorityFunction;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.Get;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.GetRequest;
@@ -65,8 +63,7 @@ public class TestPriorityRpc {
conf.setBoolean("hbase.testing.nocluster", true); // No need to do ZK
final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
TEST_UTIL.getDataTestDir(this.getClass().getName());
- CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf);
- regionServer = HRegionServer.constructRegionServer(HRegionServer.class, conf, cp);
+ regionServer = HRegionServer.constructRegionServer(HRegionServer.class, conf);
priority = regionServer.rpcServices.getPriority();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java
index edd5673..bf5787f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java
@@ -191,9 +191,9 @@ public class TestRSKilledWhenInitializing {
* notices and so removes the region from its set of online regionservers.
*/
static class RegisterAndDieRegionServer extends MiniHBaseCluster.MiniHBaseClusterRegionServer {
- public RegisterAndDieRegionServer(Configuration conf, CoordinatedStateManager cp)
+ public RegisterAndDieRegionServer(Configuration conf)
throws IOException, InterruptedException {
- super(conf, cp);
+ super(conf);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
index 15c6b76..bdcc559 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
@@ -518,10 +518,8 @@ public class TestRegionMergeTransactionOnCluster {
// Make it public so that JVMClusterUtil can access it.
public static class MyMaster extends HMaster {
- public MyMaster(Configuration conf, CoordinatedStateManager cp)
- throws IOException, KeeperException,
- InterruptedException {
- super(conf, cp);
+ public MyMaster(Configuration conf) throws IOException, KeeperException, InterruptedException {
+ super(conf);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerHostname.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerHostname.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerHostname.java
index 8eee9b6..872fec6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerHostname.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerHostname.java
@@ -71,7 +71,7 @@ public class TestRegionServerHostname {
TEST_UTIL.getConfiguration().set(HRegionServer.RS_HOSTNAME_KEY, invalidHostname);
HRegionServer hrs = null;
try {
- hrs = new HRegionServer(TEST_UTIL.getConfiguration(), null);
+ hrs = new HRegionServer(TEST_UTIL.getConfiguration());
} catch (IllegalArgumentException iae) {
assertTrue(iae.getMessage(),
iae.getMessage().contains("Failed resolve of " + invalidHostname) ||
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
index 6f8a23b..f32a87c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java
@@ -164,10 +164,9 @@ public class TestRegionServerReportForDuty {
private boolean rpcStubCreatedFlag = false;
private boolean masterChanged = false;
- public MyRegionServer(Configuration conf, CoordinatedStateManager cp)
- throws IOException, KeeperException,
+ public MyRegionServer(Configuration conf) throws IOException, KeeperException,
InterruptedException {
- super(conf, cp);
+ super(conf);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
index d3771f6..b9e77c6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
@@ -397,11 +397,6 @@ public class TestScannerHeartbeatMessages {
super(conf);
}
- public HeartbeatHRegionServer(Configuration conf, CoordinatedStateManager csm)
- throws IOException {
- super(conf, csm);
- }
-
@Override
protected RSRpcServices createRpcServices() throws IOException {
return new HeartbeatRPCServices(this);
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
index a01c75d..40077f9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.ChoreService;
import org.apache.hadoop.hbase.CoordinatedStateManager;
-import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
@@ -45,6 +44,7 @@ import org.apache.hadoop.hbase.SplitLogCounters;
import org.apache.hadoop.hbase.SplitLogTask;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.client.ClusterConnection;
+import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.executor.ExecutorType;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
@@ -88,8 +88,7 @@ public class TestSplitLogWorker {
public DummyServer(ZooKeeperWatcher zkw, Configuration conf) {
this.zkw = zkw;
this.conf = conf;
- cm = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf);
- cm.initialize(this);
+ cm = new ZkCoordinatedStateManager(this);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 1965d5a..8533004 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -951,10 +951,8 @@ public class TestSplitTransactionOnCluster {
// Make it public so that JVMClusterUtil can access it.
public static class MyMaster extends HMaster {
- public MyMaster(Configuration conf, CoordinatedStateManager cp)
- throws IOException, KeeperException,
- InterruptedException {
- super(conf, cp);
+ public MyMaster(Configuration conf) throws IOException, KeeperException, InterruptedException {
+ super(conf);
}
@Override
[05/23] hbase git commit: HBASE-19070 temporarily make the mvnsite
nightly test non-voting.
Posted by zh...@apache.org.
HBASE-19070 temporarily make the mvnsite nightly test non-voting.
Signed-off-by: Mike Drob <md...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cda2949b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cda2949b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cda2949b
Branch: refs/heads/HBASE-18410
Commit: cda2949b816fceb4d63d2d9ca445bc75912227e2
Parents: cb506fd
Author: Sean Busbey <bu...@apache.org>
Authored: Tue Oct 24 10:30:00 2017 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Tue Oct 24 12:06:58 2017 -0500
----------------------------------------------------------------------
dev-support/Jenkinsfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/cda2949b/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 28efa61..05c8895 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -49,7 +49,7 @@ pipeline {
// output from surefire; sadly the archive function in yetus only works on file names.
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt'
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
- TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop'
+ TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
EXCLUDE_TESTS_URL = 'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
}
[13/23] hbase git commit: BASE-19074 Miscellaneous Observer cleanups;
ADDEDNUM to fix FindBugs
Posted by zh...@apache.org.
BASE-19074 Miscellaneous Observer cleanups; ADDEDNUM to fix FindBugs
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/43a8ac00
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/43a8ac00
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/43a8ac00
Branch: refs/heads/HBASE-18410
Commit: 43a8ac00158e92c3015af7753edd8e835dc6054b
Parents: 95528d1
Author: Michael Stack <st...@apache.org>
Authored: Tue Oct 24 22:40:30 2017 -0700
Committer: Michael Stack <st...@apache.org>
Committed: Tue Oct 24 22:41:04 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java | 2 +-
.../java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/43a8ac00/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java
index cf2ef6f..557a61a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java
@@ -62,7 +62,7 @@ public class MemStoreSize {
@Override
public boolean equals(Object obj) {
- if (obj == null || !(obj instanceof MemStoreSize)) {
+ if (obj == null || getClass() != obj.getClass()) {
return false;
}
MemStoreSize other = (MemStoreSize) obj;
http://git-wip-us.apache.org/repos/asf/hbase/blob/43a8ac00/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java
index fade622..b13201d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java
@@ -82,7 +82,7 @@ public class MemStoreSizing extends MemStoreSize {
@Override
public boolean equals(Object obj) {
- if (obj == null || !(obj instanceof MemStoreSizing)) {
+ if (obj == null || (getClass() != obj.getClass())) {
return false;
}
MemStoreSizing other = (MemStoreSizing) obj;
[07/23] hbase git commit: HBASE-19018 tests that need bouncycastle
must delcare dependency on it.
Posted by zh...@apache.org.
HBASE-19018 tests that need bouncycastle must delcare dependency on it.
Signed-off-by: Josh Elser <el...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9353c59a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9353c59a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9353c59a
Branch: refs/heads/HBASE-18410
Commit: 9353c59a99f914716a227a1a7a2d099f92e0d06c
Parents: 3cced09
Author: Sean Busbey <bu...@apache.org>
Authored: Mon Oct 16 12:44:11 2017 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Tue Oct 24 14:30:03 2017 -0500
----------------------------------------------------------------------
hbase-endpoint/pom.xml | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9353c59a/hbase-endpoint/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-endpoint/pom.xml b/hbase-endpoint/pom.xml
index ef2584c..a127763 100644
--- a/hbase-endpoint/pom.xml
+++ b/hbase-endpoint/pom.xml
@@ -216,6 +216,12 @@
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
+ <!-- Some tests rely on Hadoop's KeyStoreTestUtil, which needs bc. -->
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk16</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<profiles>
<!-- Skip the tests in this module -->
[19/23] hbase git commit: HBASE-18160 Fix incorrect logic in
FilterList.filterKeyValue
Posted by zh...@apache.org.
HBASE-18160 Fix incorrect logic in FilterList.filterKeyValue
Signed-off-by: zhangduo <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a811fb41
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a811fb41
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a811fb41
Branch: refs/heads/HBASE-18410
Commit: a811fb410df2685253d49bcca9b4200beb279712
Parents: 1061225
Author: huzheng <op...@gmail.com>
Authored: Thu Jun 8 15:58:42 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Wed Oct 25 20:36:19 2017 +0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/filter/FilterList.java | 541 ++++++++++++-------
.../hadoop/hbase/filter/TestFilterList.java | 148 ++++-
2 files changed, 471 insertions(+), 218 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a811fb41/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index 3ff978d..3147ab0 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -90,62 +90,53 @@ final public class FilterList extends FilterBase {
private Cell transformedCell = null;
/**
- * Constructor that takes a set of {@link Filter}s. The default operator
- * MUST_PASS_ALL is assumed.
+ * Constructor that takes a set of {@link Filter}s and an operator.
+ * @param operator Operator to process filter set with.
+ * @param rowFilters Set of row filters.
+ */
+ public FilterList(final Operator operator, final List<Filter> rowFilters) {
+ reversed = checkAndGetReversed(rowFilters, reversed);
+ this.filters = new ArrayList<>(rowFilters);
+ this.operator = operator;
+ initPrevListForMustPassOne(rowFilters.size());
+ }
+
+ /**
+ * Constructor that takes a set of {@link Filter}s. The default operator MUST_PASS_ALL is assumed.
* All filters are cloned to internal list.
* @param rowFilters list of filters
*/
public FilterList(final List<Filter> rowFilters) {
- reversed = getReversed(rowFilters, reversed);
- this.filters = new ArrayList<>(rowFilters);
- initPrevListForMustPassOne(rowFilters.size());
+ this(Operator.MUST_PASS_ALL, rowFilters);
}
/**
- * Constructor that takes a var arg number of {@link Filter}s. The fefault operator
- * MUST_PASS_ALL is assumed.
+ * Constructor that takes a var arg number of {@link Filter}s. The default operator MUST_PASS_ALL
+ * is assumed.
* @param rowFilters
*/
public FilterList(final Filter... rowFilters) {
- this(Arrays.asList(rowFilters));
+ this(Operator.MUST_PASS_ALL, Arrays.asList(rowFilters));
}
/**
* Constructor that takes an operator.
- *
* @param operator Operator to process filter set with.
*/
public FilterList(final Operator operator) {
- this.operator = operator;
- this.filters = new ArrayList<>();
- initPrevListForMustPassOne(filters.size());
- }
-
- /**
- * Constructor that takes a set of {@link Filter}s and an operator.
- *
- * @param operator Operator to process filter set with.
- * @param rowFilters Set of row filters.
- */
- public FilterList(final Operator operator, final List<Filter> rowFilters) {
- this(rowFilters);
- this.operator = operator;
- initPrevListForMustPassOne(rowFilters.size());
+ this(operator, new ArrayList<>());
}
/**
* Constructor that takes a var arg number of {@link Filter}s and an operator.
- *
* @param operator Operator to process filter set with.
* @param rowFilters Filters to use
*/
public FilterList(final Operator operator, final Filter... rowFilters) {
- this(rowFilters);
- this.operator = operator;
- initPrevListForMustPassOne(rowFilters.length);
+ this(operator, Arrays.asList(rowFilters));
}
- public void initPrevListForMustPassOne(int size) {
+ private void initPrevListForMustPassOne(int size) {
if (operator == Operator.MUST_PASS_ONE) {
if (this.prevFilterRCList == null) {
prevFilterRCList = new ArrayList<>(Collections.nCopies(size, null));
@@ -156,10 +147,8 @@ final public class FilterList extends FilterBase {
}
}
-
/**
* Get the operator.
- *
* @return operator
*/
public Operator getOperator() {
@@ -168,7 +157,6 @@ final public class FilterList extends FilterBase {
/**
* Get the filters.
- *
* @return filters
*/
public List<Filter> getFilters() {
@@ -183,33 +171,22 @@ final public class FilterList extends FilterBase {
return filters.isEmpty();
}
- private static boolean getReversed(List<Filter> rowFilters, boolean defaultValue) {
- boolean rval = defaultValue;
- boolean isFirst = true;
- for (Filter f : rowFilters) {
- if (isFirst) {
- rval = f.isReversed();
- isFirst = false;
- continue;
- }
- if (rval != f.isReversed()) {
- throw new IllegalArgumentException("Filters in the list must have the same reversed flag");
- }
+ private static boolean checkAndGetReversed(List<Filter> rowFilters, boolean defaultValue) {
+ if (rowFilters.isEmpty()) {
+ return defaultValue;
}
- return rval;
- }
- private static void checkReversed(List<Filter> rowFilters, boolean expected) {
- for (Filter filter : rowFilters) {
- if (expected != filter.isReversed()) {
- throw new IllegalArgumentException(
- "Filters in the list must have the same reversed flag, expected="
- + expected);
- }
+ Boolean retValue = rowFilters.get(0).isReversed();
+ boolean allEqual = rowFilters.stream().map(Filter::isReversed).allMatch(retValue::equals);
+ if (!allEqual) {
+ throw new IllegalArgumentException("Filters in the list must have the same reversed flag");
}
+ return retValue;
}
public void addFilter(List<Filter> filters) {
- checkReversed(filters, isReversed());
+ if (checkAndGetReversed(filters, isReversed()) != isReversed()) {
+ throw new IllegalArgumentException("Filters in the list must have the same reversed flag");
+ }
this.filters.addAll(filters);
if (operator == Operator.MUST_PASS_ONE) {
this.prevFilterRCList.addAll(Collections.nCopies(filters.size(), null));
@@ -219,7 +196,6 @@ final public class FilterList extends FilterBase {
/**
* Add a filter.
- *
* @param filter another filter
*/
public void addFilter(Filter filter) {
@@ -228,8 +204,7 @@ final public class FilterList extends FilterBase {
@Override
public void reset() throws IOException {
- int listize = filters.size();
- for (int i = 0; i < listize; i++) {
+ for (int i = 0, n = filters.size(); i < n; i++) {
filters.get(i).reset();
if (operator == Operator.MUST_PASS_ONE) {
prevFilterRCList.set(i, null);
@@ -245,18 +220,15 @@ final public class FilterList extends FilterBase {
return super.filterRowKey(rowKey, offset, length);
}
boolean flag = this.operator == Operator.MUST_PASS_ONE;
- int listize = filters.size();
- for (int i = 0; i < listize; i++) {
+ for (int i = 0, n = filters.size(); i < n; i++) {
Filter filter = filters.get(i);
if (this.operator == Operator.MUST_PASS_ALL) {
- if (filter.filterAllRemaining() ||
- filter.filterRowKey(rowKey, offset, length)) {
- flag = true;
+ if (filter.filterAllRemaining() || filter.filterRowKey(rowKey, offset, length)) {
+ flag = true;
}
} else if (this.operator == Operator.MUST_PASS_ONE) {
- if (!filter.filterAllRemaining() &&
- !filter.filterRowKey(rowKey, offset, length)) {
- flag = false;
+ if (!filter.filterAllRemaining() && !filter.filterRowKey(rowKey, offset, length)) {
+ flag = false;
}
}
}
@@ -269,8 +241,7 @@ final public class FilterList extends FilterBase {
return super.filterRowKey(firstRowCell);
}
boolean flag = this.operator == Operator.MUST_PASS_ONE;
- int listize = filters.size();
- for (int i = 0; i < listize; i++) {
+ for (int i = 0, n = filters.size(); i < n; i++) {
Filter filter = filters.get(i);
if (this.operator == Operator.MUST_PASS_ALL) {
if (filter.filterAllRemaining() || filter.filterRowKey(firstRowCell)) {
@@ -290,8 +261,7 @@ final public class FilterList extends FilterBase {
if (isEmpty()) {
return super.filterAllRemaining();
}
- int listize = filters.size();
- for (int i = 0; i < listize; i++) {
+ for (int i = 0, n = filters.size(); i < n; i++) {
if (filters.get(i).filterAllRemaining()) {
if (operator == Operator.MUST_PASS_ALL) {
return true;
@@ -311,10 +281,15 @@ final public class FilterList extends FilterBase {
return super.transformCell(c);
}
if (!CellUtil.equals(c, referenceCell)) {
- throw new IllegalStateException("Reference Cell: " + this.referenceCell + " does not match: "
- + c);
+ throw new IllegalStateException(
+ "Reference Cell: " + this.referenceCell + " does not match: " + c);
}
- return this.transformedCell;
+ // Copy transformedCell into a new cell and reset transformedCell & referenceCell to null for
+ // Java GC optimization
+ Cell cell = KeyValueUtil.copyToNewKeyValue(this.transformedCell);
+ this.transformedCell = null;
+ this.referenceCell = null;
+ return cell;
}
/**
@@ -352,116 +327,315 @@ final public class FilterList extends FilterBase {
}
}
- @Override
- @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="SF_SWITCH_FALLTHROUGH",
- justification="Intentional")
- public ReturnCode filterKeyValue(Cell c) throws IOException {
- if (isEmpty()) {
- return ReturnCode.INCLUDE;
+ /**
+ * FilterList with MUST_PASS_ALL choose the maximal forward step among sub-filters in filter list.
+ * Let's call it: The Maximal Step Rule. So if filter-A in filter list return INCLUDE and filter-B
+ * in filter list return INCLUDE_AND_NEXT_COL, then the filter list should return
+ * INCLUDE_AND_NEXT_COL. For SEEK_NEXT_USING_HINT, it's more special, and in method
+ * filterKeyValueWithMustPassAll(), if any sub-filter return SEEK_NEXT_USING_HINT, then our filter
+ * list will return SEEK_NEXT_USING_HINT. so we don't care about the SEEK_NEXT_USING_HINT here. <br/>
+ * <br/>
+ * The jump step will be:
+ *
+ * <pre>
+ * INCLUDE < SKIP < INCLUDE_AND_NEXT_COL < NEXT_COL < INCLUDE_AND_SEEK_NEXT_ROW < NEXT_ROW < SEEK_NEXT_USING_HINT
+ * </pre>
+ *
+ * Here, we have the following map to describe The Maximal Step Rule. if current return code (for
+ * previous sub-filters in filter list) is <strong>ReturnCode</strong>, and current filter returns
+ * <strong>localRC</strong>, then we should return map[ReturnCode][localRC] for the merged result,
+ * according to The Maximal Step Rule. <br/>
+ *
+ * <pre>
+ * LocalCode\ReturnCode INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW SKIP NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
+ * INCLUDE INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW SKIP NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
+ * INCLUDE_AND_NEXT_COL INCLUDE_AND_NEXT_COL INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW NEXT_COL NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
+ * INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW NEXT_ROW NEXT_ROW NEXT_ROW SEEK_NEXT_USING_HINT
+ * SKIP SKIP NEXT_COL NEXT_ROW SKIP NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
+ * NEXT_COL NEXT_COL NEXT_COL NEXT_ROW NEXT_COL NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
+ * NEXT_ROW NEXT_ROW NEXT_ROW NEXT_ROW NEXT_ROW NEXT_ROW NEXT_ROW SEEK_NEXT_USING_HINT
+ * SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT
+ * </pre>
+ * @param rc Return code which is calculated by previous sub-filter(s) in filter list.
+ * @param localRC Return code of the current sub-filter in filter list.
+ * @return Return code which is merged by the return code of previous sub-filter(s) and the return
+ * code of current sub-filter.
+ */
+ private ReturnCode mergeReturnCodeForAndOperator(ReturnCode rc, ReturnCode localRC) {
+ if (rc == ReturnCode.SEEK_NEXT_USING_HINT || localRC == ReturnCode.SEEK_NEXT_USING_HINT) {
+ return ReturnCode.SEEK_NEXT_USING_HINT;
}
- this.referenceCell = c;
- seekHintFilters.clear();
+ switch (localRC) {
+ case INCLUDE:
+ return rc;
+ case INCLUDE_AND_NEXT_COL:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL)) {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL)) {
+ return ReturnCode.NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.NEXT_ROW;
+ }
+ break;
+ case INCLUDE_AND_SEEK_NEXT_ROW:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
+ ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.NEXT_ROW;
+ }
+ break;
+ case SKIP:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP)) {
+ return ReturnCode.SKIP;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.NEXT_COL)) {
+ return ReturnCode.NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.NEXT_ROW;
+ }
+ break;
+ case NEXT_COL:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.SKIP,
+ ReturnCode.NEXT_COL)) {
+ return ReturnCode.NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.NEXT_ROW;
+ }
+ break;
+ case NEXT_ROW:
+ return ReturnCode.NEXT_ROW;
+ }
+ throw new IllegalStateException("Received code is not valid. rc: " + rc + ", localRC: "
+ + localRC);
+ }
- // Accumulates successive transformation of every filter that includes the Cell:
+ private ReturnCode filterKeyValueWithMustPassAll(Cell c) throws IOException {
+ ReturnCode rc = ReturnCode.INCLUDE;
Cell transformed = c;
+ this.seekHintFilters.clear();
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ Filter filter = filters.get(i);
+ if (filter.filterAllRemaining()) {
+ return ReturnCode.NEXT_ROW;
+ }
+ ReturnCode localRC = filter.filterKeyValue(c);
+ rc = mergeReturnCodeForAndOperator(rc, localRC);
- ReturnCode rc = operator == Operator.MUST_PASS_ONE?
- ReturnCode.SKIP: ReturnCode.INCLUDE;
- int listize = filters.size();
- /*
- * When all filters in a MUST_PASS_ONE FilterList return a SEEK_USING_NEXT_HINT code,
- * we should return SEEK_NEXT_USING_HINT from the FilterList to utilize the lowest seek value.
- *
- * The following variable tracks whether any of the Filters returns ReturnCode other than
- * SEEK_NEXT_USING_HINT for MUST_PASS_ONE FilterList, in which case the optimization would
- * be skipped.
- */
- boolean seenNonHintReturnCode = false;
- for (int i = 0; i < listize; i++) {
+ // For INCLUDE* case, we need to update the transformed cell.
+ if (isInReturnCodes(localRC, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
+ ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ transformed = filter.transformCell(transformed);
+ }
+ if (localRC == ReturnCode.SEEK_NEXT_USING_HINT) {
+ seekHintFilters.add(filter);
+ }
+ }
+ this.transformedCell = transformed;
+ if (!seekHintFilters.isEmpty()) {
+ return ReturnCode.SEEK_NEXT_USING_HINT;
+ }
+ return rc;
+ }
+
+ private void updatePrevFilterRCList(int index, ReturnCode currentRC) {
+ prevFilterRCList.set(index, currentRC);
+ }
+
+ private void updatePrevCellList(int index, Cell currentCell, ReturnCode currentRC) {
+ if (currentCell == null || currentRC == ReturnCode.INCLUDE || currentRC == ReturnCode.SKIP) {
+ // If previous return code is INCLUDE or SKIP, we should always pass the next cell to the
+ // corresponding sub-filter(need not test shouldPassCurrentCellToFilter() method), So we
+ // need not save current cell to prevCellList for saving heap memory.
+ prevCellList.set(index, null);
+ } else {
+ prevCellList.set(index, KeyValueUtil.toNewKeyCell(currentCell));
+ }
+ }
+
+ private static boolean isInReturnCodes(ReturnCode testRC, ReturnCode... returnCodes) {
+ return Arrays.stream(returnCodes).anyMatch(testRC::equals);
+ }
+
+ /**
+ * FilterList with MUST_PASS_ONE choose the minimal forward step among sub-filter in filter list.
+ * Let's call it: The Minimal Step Rule. So if filter-A in filter list return INCLUDE and filter-B
+ * in filter list return INCLUDE_AND_NEXT_COL, then the filter list should return INCLUDE. For
+ * SEEK_NEXT_USING_HINT, it's more special, because we do not know how far it will forward, so we
+ * use SKIP by default.<br/>
+ * <br/>
+ * The jump step will be:
+ *
+ * <pre>
+ * INCLUDE < SKIP < INCLUDE_AND_NEXT_COL < NEXT_COL < INCLUDE_AND_SEEK_NEXT_ROW < NEXT_ROW < SEEK_NEXT_USING_HINT
+ * </pre>
+ *
+ * Here, we have the following map to describe The Minimal Step Rule. if current return code (for
+ * previous sub-filters in filter list) is <strong>ReturnCode</strong>, and current filter returns
+ * <strong>localRC</strong>, then we should return map[ReturnCode][localRC] for the merged result,
+ * according to The Minimal Step Rule.<br/>
+ *
+ * <pre>
+ * LocalCode\ReturnCode INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW SKIP NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
+ * INCLUDE INCLUDE INCLUDE INCLUDE INCLUDE INCLUDE INCLUDE INCLUDE
+ * INCLUDE_AND_NEXT_COL INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_NEXT_COL INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_NEXT_COL INCLUDE
+ * INCLUDE_AND_SEEK_NEXT_ROW INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW INCLUDE
+ * SKIP INCLUDE INCLUDE INCLUDE SKIP SKIP SKIP SKIP
+ * NEXT_COL INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_NEXT_COL SKIP NEXT_COL NEXT_COL SKIP
+ * NEXT_ROW INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW SKIP NEXT_COL NEXT_ROW SKIP
+ * SEEK_NEXT_USING_HINT INCLUDE INCLUDE INCLUDE SKIP SKIP SKIP SEEK_NEXT_USING_HINT
+ * </pre>
+ * @param rc Return code which is calculated by previous sub-filter(s) in filter list.
+ * @param localRC Return code of the current sub-filter in filter list.
+ * @return Return code which is merged by the return code of previous sub-filter(s) and the return
+ * code of current sub-filter.
+ */
+ private ReturnCode mergeReturnCodeForOrOperator(ReturnCode rc, ReturnCode localRC) {
+ if (rc == null) return localRC;
+ switch (localRC) {
+ case INCLUDE:
+ return ReturnCode.INCLUDE;
+ case INCLUDE_AND_NEXT_COL:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP,
+ ReturnCode.SEEK_NEXT_USING_HINT)) {
+ return ReturnCode.INCLUDE;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW,
+ ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ break;
+ case INCLUDE_AND_SEEK_NEXT_ROW:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP,
+ ReturnCode.SEEK_NEXT_USING_HINT)) {
+ return ReturnCode.INCLUDE;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.NEXT_COL)) {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
+ }
+ break;
+ case SKIP:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
+ ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ return ReturnCode.INCLUDE;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW,
+ ReturnCode.SEEK_NEXT_USING_HINT)) {
+ return ReturnCode.SKIP;
+ }
+ break;
+ case NEXT_COL:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE)) {
+ return ReturnCode.INCLUDE;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL,
+ ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.SEEK_NEXT_USING_HINT)) {
+ return ReturnCode.SKIP;
+ }
+ break;
+ case NEXT_ROW:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE)) {
+ return ReturnCode.INCLUDE;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL)) {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.SEEK_NEXT_USING_HINT)) {
+ return ReturnCode.SKIP;
+ }
+ if (isInReturnCodes(rc, ReturnCode.NEXT_COL)) {
+ return ReturnCode.NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.NEXT_ROW;
+ }
+ case SEEK_NEXT_USING_HINT:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
+ ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ return ReturnCode.INCLUDE;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.SKIP;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SEEK_NEXT_USING_HINT)) {
+ return ReturnCode.SEEK_NEXT_USING_HINT;
+ }
+ }
+ throw new IllegalStateException(
+ "Received code is not valid. rc: " + rc + ", localRC: " + localRC);
+ }
+
+ private ReturnCode filterKeyValueWithMustPassOne(Cell c) throws IOException {
+ ReturnCode rc = null;
+ boolean everyFilterReturnHint = true;
+ Cell transformed = c;
+ for (int i = 0, n = filters.size(); i < n; i++) {
Filter filter = filters.get(i);
- if (operator == Operator.MUST_PASS_ALL) {
- if (filter.filterAllRemaining()) {
- return ReturnCode.NEXT_ROW;
- }
- ReturnCode code = filter.filterKeyValue(c);
- switch (code) {
- // Override INCLUDE and continue to evaluate.
- case INCLUDE_AND_NEXT_COL:
- rc = ReturnCode.INCLUDE_AND_NEXT_COL; // FindBugs SF_SWITCH_FALLTHROUGH
- case INCLUDE:
- transformed = filter.transformCell(transformed);
- continue;
- case SEEK_NEXT_USING_HINT:
- seekHintFilters.add(filter);
- continue;
- default:
- if (seekHintFilters.isEmpty()) {
- return code;
- }
- }
- } else if (operator == Operator.MUST_PASS_ONE) {
- Cell prevCell = this.prevCellList.get(i);
- if (filter.filterAllRemaining() || !shouldPassCurrentCellToFilter(prevCell, c, i)) {
- seenNonHintReturnCode = true;
- continue;
- }
+ Cell prevCell = this.prevCellList.get(i);
+ if (filter.filterAllRemaining() || !shouldPassCurrentCellToFilter(prevCell, c, i)) {
+ everyFilterReturnHint = false;
+ continue;
+ }
- ReturnCode localRC = filter.filterKeyValue(c);
- // Update previous cell and return code we encountered.
- prevFilterRCList.set(i, localRC);
- if (c == null || localRC == ReturnCode.INCLUDE || localRC == ReturnCode.SKIP) {
- // If previous return code is INCLUDE or SKIP, we should always pass the next cell to the
- // corresponding sub-filter(need not test shouldPassCurrentCellToFilter() method), So we
- // need not save current cell to prevCellList for saving heap memory.
- prevCellList.set(i, null);
- } else {
- prevCellList.set(i, KeyValueUtil.toNewKeyCell(c));
- }
+ ReturnCode localRC = filter.filterKeyValue(c);
- if (localRC != ReturnCode.SEEK_NEXT_USING_HINT) {
- seenNonHintReturnCode = true;
- }
- switch (localRC) {
- case INCLUDE:
- if (rc != ReturnCode.INCLUDE_AND_NEXT_COL) {
- rc = ReturnCode.INCLUDE;
- }
- transformed = filter.transformCell(transformed);
- break;
- case INCLUDE_AND_NEXT_COL:
- rc = ReturnCode.INCLUDE_AND_NEXT_COL;
- transformed = filter.transformCell(transformed);
- // must continue here to evaluate all filters
- break;
- case NEXT_ROW:
- break;
- case SKIP:
- break;
- case NEXT_COL:
- break;
- case SEEK_NEXT_USING_HINT:
- break;
- default:
- throw new IllegalStateException("Received code is not valid.");
- }
+ // Update previous return code and previous cell for filter[i].
+ updatePrevFilterRCList(i, localRC);
+ updatePrevCellList(i, c, localRC);
+
+ if (localRC != ReturnCode.SEEK_NEXT_USING_HINT) {
+ everyFilterReturnHint = false;
+ }
+
+ rc = mergeReturnCodeForOrOperator(rc, localRC);
+
+ // For INCLUDE* case, we need to update the transformed cell.
+ if (isInReturnCodes(localRC, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
+ ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ transformed = filter.transformCell(transformed);
}
}
- if (!seekHintFilters.isEmpty()) {
+ this.transformedCell = transformed;
+ if (everyFilterReturnHint) {
return ReturnCode.SEEK_NEXT_USING_HINT;
+ } else if (rc == null) {
+ // Each sub-filter in filter list got true for filterAllRemaining().
+ return ReturnCode.SKIP;
+ } else {
+ return rc;
}
+ }
- // Save the transformed Cell for transform():
- this.transformedCell = transformed;
+ @Override
+ public ReturnCode filterKeyValue(Cell c) throws IOException {
+ if (isEmpty()) {
+ return ReturnCode.INCLUDE;
+ }
+ this.referenceCell = c;
- /*
- * The seenNonHintReturnCode flag is intended only for Operator.MUST_PASS_ONE branch.
- * If we have seen non SEEK_NEXT_USING_HINT ReturnCode, respect that ReturnCode.
- */
- if (operator == Operator.MUST_PASS_ONE && !seenNonHintReturnCode) {
- return ReturnCode.SEEK_NEXT_USING_HINT;
+ if (operator == Operator.MUST_PASS_ALL) {
+ return filterKeyValueWithMustPassAll(c);
+ } else {
+ return filterKeyValueWithMustPassOne(c);
}
- return rc;
}
/**
@@ -472,16 +646,14 @@ final public class FilterList extends FilterBase {
*/
@Override
public void filterRowCells(List<Cell> cells) throws IOException {
- int listize = filters.size();
- for (int i = 0; i < listize; i++) {
+ for (int i = 0, n = filters.size(); i < n; i++) {
filters.get(i).filterRowCells(cells);
}
}
@Override
public boolean hasFilterRow() {
- int listize = filters.size();
- for (int i = 0; i < listize; i++) {
+ for (int i = 0, n = filters.size(); i < n; i++) {
if (filters.get(i).hasFilterRow()) {
return true;
}
@@ -494,8 +666,7 @@ final public class FilterList extends FilterBase {
if (isEmpty()) {
return super.filterRow();
}
- int listize = filters.size();
- for (int i = 0; i < listize; i++) {
+ for (int i = 0, n = filters.size(); i < n; i++) {
Filter filter = filters.get(i);
if (operator == Operator.MUST_PASS_ALL) {
if (filter.filterRow()) {
@@ -507,18 +678,16 @@ final public class FilterList extends FilterBase {
}
}
}
- return operator == Operator.MUST_PASS_ONE;
+ return operator == Operator.MUST_PASS_ONE;
}
/**
* @return The filter serialized using pb
*/
public byte[] toByteArray() throws IOException {
- FilterProtos.FilterList.Builder builder =
- FilterProtos.FilterList.newBuilder();
+ FilterProtos.FilterList.Builder builder = FilterProtos.FilterList.newBuilder();
builder.setOperator(FilterProtos.FilterList.Operator.valueOf(operator.name()));
- int listize = filters.size();
- for (int i = 0; i < listize; i++) {
+ for (int i = 0, n = filters.size(); i < n; i++) {
builder.addFilters(ProtobufUtil.toFilter(filters.get(i)));
}
return builder.build().toByteArray();
@@ -531,7 +700,7 @@ final public class FilterList extends FilterBase {
* @see #toByteArray
*/
public static FilterList parseFrom(final byte [] pbBytes)
- throws DeserializationException {
+ throws DeserializationException {
FilterProtos.FilterList proto;
try {
proto = FilterProtos.FilterList.parseFrom(pbBytes);
@@ -542,14 +711,13 @@ final public class FilterList extends FilterBase {
List<Filter> rowFilters = new ArrayList<>(proto.getFiltersCount());
try {
List<FilterProtos.Filter> filtersList = proto.getFiltersList();
- int listSize = filtersList.size();
- for (int i = 0; i < listSize; i++) {
+ for (int i = 0, n = filtersList.size(); i < n; i++) {
rowFilters.add(ProtobufUtil.toFilter(filtersList.get(i)));
}
} catch (IOException ioe) {
throw new DeserializationException(ioe);
}
- return new FilterList(Operator.valueOf(proto.getOperator().name()),rowFilters);
+ return new FilterList(Operator.valueOf(proto.getOperator().name()), rowFilters);
}
/**
@@ -589,8 +757,7 @@ final public class FilterList extends FilterBase {
}
// If any condition can pass, we need to keep the min hint
- int listize = filters.size();
- for (int i = 0; i < listize; i++) {
+ for (int i = 0, n = filters.size(); i < n; i++) {
if (filters.get(i).filterAllRemaining()) {
continue;
}
@@ -616,8 +783,7 @@ final public class FilterList extends FilterBase {
if (isEmpty()) {
return super.isFamilyEssential(name);
}
- int listize = filters.size();
- for (int i = 0; i < listize; i++) {
+ for (int i = 0, n = filters.size(); i < n; i++) {
if (filters.get(i).isFamilyEssential(name)) {
return true;
}
@@ -627,8 +793,7 @@ final public class FilterList extends FilterBase {
@Override
public void setReversed(boolean reversed) {
- int listize = filters.size();
- for (int i = 0; i < listize; i++) {
+ for (int i = 0, n = filters.size(); i < n; i++) {
filters.get(i).setReversed(reversed);
}
this.reversed = reversed;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a811fb41/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index e414729..f7cba18 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -24,6 +24,8 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import com.google.common.collect.Lists;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -43,21 +45,12 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-/**
- * Tests filter sets
- *
- */
+
@Category({FilterTests.class, SmallTests.class})
public class TestFilterList {
static final int MAX_PAGES = 2;
- static final char FIRST_CHAR = 'a';
- static final char LAST_CHAR = 'e';
- static byte[] GOOD_BYTES = Bytes.toBytes("abc");
- static byte[] BAD_BYTES = Bytes.toBytes("def");
-
@Test
public void testAddFilter() throws Exception {
@@ -311,23 +304,23 @@ public class TestFilterList {
FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
flist.addFilter(new PrefixFilter(r1));
flist.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
- assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE);
- assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE);
+ assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r1, r1, r1)));
+ assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r11, r11, r11)));
flist.reset();
flist.filterRowKey(KeyValueUtil.createFirstOnRow(r2));
- assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
+ assertEquals(ReturnCode.SKIP, flist.filterKeyValue(new KeyValue(r2, r2, r2)));
flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
flist.addFilter(new AlwaysNextColFilter());
flist.addFilter(new PrefixFilter(r1));
flist.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
- assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE);
- assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE);
+ assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r1, r1, r1)));
+ assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r11, r11, r11)));
flist.reset();
flist.filterRowKey(KeyValueUtil.createFirstOnRow(r2));
- assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
+ assertEquals(ReturnCode.NEXT_COL, flist.filterKeyValue(new KeyValue(r2, r2, r2)));
}
/**
@@ -345,12 +338,12 @@ public class TestFilterList {
flist.addFilter(new AlwaysNextColFilter());
flist.addFilter(new InclusiveStopFilter(r1));
flist.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
- assertEquals(flist.filterKeyValue(new KeyValue(r1,r1,r1)), ReturnCode.INCLUDE);
- assertEquals(flist.filterKeyValue(new KeyValue(r11,r11,r11)), ReturnCode.INCLUDE);
+ assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r1, r1, r1)));
+ assertEquals(ReturnCode.INCLUDE, flist.filterKeyValue(new KeyValue(r11, r11, r11)));
flist.reset();
flist.filterRowKey(KeyValueUtil.createFirstOnRow(r2));
- assertEquals(flist.filterKeyValue(new KeyValue(r2,r2,r2)), ReturnCode.SKIP);
+ assertEquals(ReturnCode.NEXT_COL, flist.filterKeyValue(new KeyValue(r2, r2, r2)));
}
public static class AlwaysNextColFilter extends FilterBase {
@@ -432,7 +425,7 @@ public class TestFilterList {
FilterList mpOnefilterList = new FilterList(Operator.MUST_PASS_ONE,
Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter }));
// INCLUDE, INCLUDE, INCLUDE_AND_NEXT_COL.
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpOnefilterList.filterKeyValue(null));
+ assertEquals(ReturnCode.INCLUDE, mpOnefilterList.filterKeyValue(null));
// INCLUDE, SKIP, INCLUDE.
assertEquals(Filter.ReturnCode.INCLUDE, mpOnefilterList.filterKeyValue(null));
@@ -607,16 +600,16 @@ public class TestFilterList {
KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
3, Bytes.toBytes("value"));
- assertEquals(filterList01.filterKeyValue(kv1), ReturnCode.INCLUDE_AND_NEXT_COL);
- assertEquals(filterList01.filterKeyValue(kv2), ReturnCode.SKIP);
- assertEquals(filterList01.filterKeyValue(kv3), ReturnCode.SKIP);
+ assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList01.filterKeyValue(kv1));
+ assertEquals(ReturnCode.SKIP, filterList01.filterKeyValue(kv2));
+ assertEquals(ReturnCode.SKIP, filterList01.filterKeyValue(kv3));
FilterList filterList11 =
new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, 1));
- assertEquals(filterList11.filterKeyValue(kv1), ReturnCode.SKIP);
- assertEquals(filterList11.filterKeyValue(kv2), ReturnCode.SKIP);
- assertEquals(filterList11.filterKeyValue(kv3), ReturnCode.SKIP);
+ assertEquals(ReturnCode.NEXT_COL, filterList11.filterKeyValue(kv1));
+ assertEquals(ReturnCode.SKIP, filterList11.filterKeyValue(kv2));
+ assertEquals(ReturnCode.SKIP, filterList11.filterKeyValue(kv3));
}
@Test
@@ -634,10 +627,10 @@ public class TestFilterList {
KeyValue kv4 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 4,
Bytes.toBytes("value"));
- assertEquals(filterList.filterKeyValue(kv1), ReturnCode.SEEK_NEXT_USING_HINT);
- assertEquals(filterList.filterKeyValue(kv2), ReturnCode.SKIP);
- assertEquals(filterList.filterKeyValue(kv3), ReturnCode.INCLUDE_AND_NEXT_COL);
- assertEquals(filterList.filterKeyValue(kv4), ReturnCode.SKIP);
+ assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1));
+ assertEquals(ReturnCode.SKIP, filterList.filterKeyValue(kv2));
+ assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterKeyValue(kv3));
+ assertEquals(ReturnCode.SKIP, filterList.filterKeyValue(kv4));
}
private static class MockFilter extends FilterBase {
@@ -710,6 +703,101 @@ public class TestFilterList {
mockFilter.didCellPassToTheFilter = false;
filter.filterKeyValue(kv4);
assertTrue(mockFilter.didCellPassToTheFilter);
+
+ mockFilter = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
+ filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter);
+ filter.filterKeyValue(kv1);
+ assertTrue(mockFilter.didCellPassToTheFilter);
+
+ mockFilter.didCellPassToTheFilter = false;
+ filter.filterKeyValue(kv2);
+ assertFalse(mockFilter.didCellPassToTheFilter);
+
+ mockFilter.didCellPassToTheFilter = false;
+ filter.filterKeyValue(kv3);
+ assertFalse(mockFilter.didCellPassToTheFilter);
+
+ mockFilter.didCellPassToTheFilter = false;
+ filter.filterKeyValue(kv4);
+ assertTrue(mockFilter.didCellPassToTheFilter);
+ }
+
+ @Test
+ public void testTheMaximalRule() throws IOException {
+ KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
+ Bytes.toBytes("value"));
+ MockFilter filter1 = new MockFilter(ReturnCode.INCLUDE);
+ MockFilter filter2 = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
+ MockFilter filter3 = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
+ MockFilter filter4 = new MockFilter(ReturnCode.NEXT_COL);
+ MockFilter filter5 = new MockFilter(ReturnCode.SKIP);
+ MockFilter filter6 = new MockFilter(ReturnCode.SEEK_NEXT_USING_HINT);
+ MockFilter filter7 = new MockFilter(ReturnCode.NEXT_ROW);
+
+ FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
+ assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterKeyValue(kv1));
+
+ filterList = new FilterList(Operator.MUST_PASS_ALL, filter2, filter3);
+ assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterKeyValue(kv1));
+
+ filterList = new FilterList(Operator.MUST_PASS_ALL, filter4, filter5, filter6);
+ assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1));
+
+ filterList = new FilterList(Operator.MUST_PASS_ALL, filter4, filter6);
+ assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1));
+
+ filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter1);
+ assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterKeyValue(kv1));
+
+ filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter2, filter1, filter5);
+ assertEquals(ReturnCode.NEXT_ROW, filterList.filterKeyValue(kv1));
+
+ filterList = new FilterList(Operator.MUST_PASS_ALL, filter2,
+ new FilterList(Operator.MUST_PASS_ALL, filter3, filter4));
+ assertEquals(ReturnCode.NEXT_ROW, filterList.filterKeyValue(kv1));
+
+ filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter7);
+ assertEquals(ReturnCode.NEXT_ROW, filterList.filterKeyValue(kv1));
+ }
+
+ @Test
+ public void testTheMinimalRule() throws IOException {
+ KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
+ Bytes.toBytes("value"));
+ MockFilter filter1 = new MockFilter(ReturnCode.INCLUDE);
+ MockFilter filter2 = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
+ MockFilter filter3 = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
+ MockFilter filter4 = new MockFilter(ReturnCode.NEXT_COL);
+ MockFilter filter5 = new MockFilter(ReturnCode.SKIP);
+ MockFilter filter6 = new MockFilter(ReturnCode.SEEK_NEXT_USING_HINT);
+ FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2);
+ assertEquals(filterList.filterKeyValue(kv1), ReturnCode.INCLUDE);
+
+ filterList = new FilterList(Operator.MUST_PASS_ONE, filter2, filter3);
+ assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterKeyValue(kv1));
+
+ filterList = new FilterList(Operator.MUST_PASS_ONE, filter4, filter5, filter6);
+ assertEquals(ReturnCode.SKIP, filterList.filterKeyValue(kv1));
+
+ filterList = new FilterList(Operator.MUST_PASS_ONE, filter4, filter6);
+ assertEquals(ReturnCode.SKIP, filterList.filterKeyValue(kv1));
+
+ filterList = new FilterList(Operator.MUST_PASS_ONE, filter3, filter1);
+ assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(kv1));
+
+ filterList = new FilterList(Operator.MUST_PASS_ONE, filter3, filter2, filter1, filter5);
+ assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(kv1));
+
+ filterList = new FilterList(Operator.MUST_PASS_ONE, filter2,
+ new FilterList(Operator.MUST_PASS_ONE, filter3, filter4));
+ assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterKeyValue(kv1));
+
+ filterList = new FilterList(Operator.MUST_PASS_ONE, filter2,
+ new FilterList(Operator.MUST_PASS_ONE, filter3, filter4));
+ assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterKeyValue(kv1));
+
+ filterList = new FilterList(Operator.MUST_PASS_ONE, filter6, filter6);
+ assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1));
}
}
[17/23] hbase git commit: HBASE-18410 disable the HBASE-18957 test
until we can fix it on the feature branch.
Posted by zh...@apache.org.
HBASE-18410 disable the HBASE-18957 test until we can fix it on the feature branch.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4e1c7208
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4e1c7208
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4e1c7208
Branch: refs/heads/HBASE-18410
Commit: 4e1c7208cbbe27a926a192a9e9526c0d02d0c516
Parents: 43a8ac0
Author: Sean Busbey <bu...@apache.org>
Authored: Mon Oct 9 15:24:00 2017 -0500
Committer: zhangduo <zh...@apache.org>
Committed: Wed Oct 25 20:36:19 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/filter/TestFilterListOnMini.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4e1c7208/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterListOnMini.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterListOnMini.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterListOnMini.java
index dd2399f..590b26e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterListOnMini.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterListOnMini.java
@@ -58,6 +58,7 @@ public class TestFilterListOnMini {
TEST_UTIL.shutdownMiniCluster();
}
+ @Ignore("HBASE-18410 Should not merge without this test running.")
@Test
public void testFiltersWithOR() throws Exception {
TableName tn = TableName.valueOf(name.getMethodName());
[18/23] hbase git commit: HBASE-18411 Dividing FiterList into two
separate sub-classes: FilterListWithOR , FilterListWithAND
Posted by zh...@apache.org.
HBASE-18411 Dividing FiterList into two separate sub-classes: FilterListWithOR , FilterListWithAND
Signed-off-by: zhangduo <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/59275100
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/59275100
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/59275100
Branch: refs/heads/HBASE-18410
Commit: 59275100e1d7889e73169ab0d0a70c710e260b7a
Parents: 6442d30
Author: huzheng <op...@gmail.com>
Authored: Tue Oct 10 20:01:48 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Wed Oct 25 20:36:19 2017 +0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/filter/FilterList.java | 661 ++-----------------
.../hadoop/hbase/filter/FilterListBase.java | 159 +++++
.../hadoop/hbase/filter/FilterListWithAND.java | 273 ++++++++
.../hadoop/hbase/filter/FilterListWithOR.java | 383 +++++++++++
.../hadoop/hbase/filter/TestFilterList.java | 89 +++
5 files changed, 962 insertions(+), 603 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/59275100/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index b518645..97392d1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.yetus.audience.InterfaceAudience;
@@ -37,86 +38,60 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.FilterProtos;
/**
- * Implementation of {@link Filter} that represents an ordered List of Filters
- * which will be evaluated with a specified boolean operator {@link Operator#MUST_PASS_ALL}
- * (<code>AND</code>) or {@link Operator#MUST_PASS_ONE} (<code>OR</code>).
- * Since you can use Filter Lists as children of Filter Lists, you can create a
- * hierarchy of filters to be evaluated.
- *
- * <br>
- * {@link Operator#MUST_PASS_ALL} evaluates lazily: evaluation stops as soon as one filter does
- * not include the KeyValue.
- *
- * <br>
- * {@link Operator#MUST_PASS_ONE} evaluates non-lazily: all filters are always evaluated.
- *
- * <br>
+ * Implementation of {@link Filter} that represents an ordered List of Filters which will be
+ * evaluated with a specified boolean operator {@link Operator#MUST_PASS_ALL} (<code>AND</code>) or
+ * {@link Operator#MUST_PASS_ONE} (<code>OR</code>). Since you can use Filter Lists as children of
+ * Filter Lists, you can create a hierarchy of filters to be evaluated. <br>
+ * {@link Operator#MUST_PASS_ALL} evaluates lazily: evaluation stops as soon as one filter does not
+ * include the KeyValue. <br>
+ * {@link Operator#MUST_PASS_ONE} evaluates non-lazily: all filters are always evaluated. <br>
* Defaults to {@link Operator#MUST_PASS_ALL}.
*/
@InterfaceAudience.Public
final public class FilterList extends FilterBase {
+
/** set operator */
@InterfaceAudience.Public
- public static enum Operator {
+ public enum Operator {
/** !AND */
MUST_PASS_ALL,
/** !OR */
MUST_PASS_ONE
}
- private static final int MAX_LOG_FILTERS = 5;
- private Operator operator = Operator.MUST_PASS_ALL;
- private final List<Filter> filters;
- private Collection<Filter> seekHintFilters = new ArrayList<Filter>();
-
- /**
- * Save previous return code and previous cell for every filter in filter list. For MUST_PASS_ONE,
- * we use the previous return code to decide whether we should pass current cell encountered to
- * the filter. For MUST_PASS_ALL, the two list are meaningless.
- */
- private List<ReturnCode> prevFilterRCList = null;
- private List<Cell> prevCellList = null;
-
- /** Reference Cell used by {@link #transformCell(Cell)} for validation purpose. */
- private Cell referenceCell = null;
-
- /**
- * When filtering a given Cell in {@link #filterKeyValue(Cell)},
- * this stores the transformed Cell to be returned by {@link #transformCell(Cell)}.
- *
- * Individual filters transformation are applied only when the filter includes the Cell.
- * Transformations are composed in the order specified by {@link #filters}.
- */
- private Cell transformedCell = null;
+ private Operator operator;
+ private FilterListBase filterListBase;
/**
* Constructor that takes a set of {@link Filter}s and an operator.
* @param operator Operator to process filter set with.
- * @param rowFilters Set of row filters.
+ * @param filters Set of row filters.
*/
- public FilterList(final Operator operator, final List<Filter> rowFilters) {
- reversed = checkAndGetReversed(rowFilters, reversed);
- this.filters = new ArrayList<>(rowFilters);
+ public FilterList(final Operator operator, final List<Filter> filters) {
+ if (operator == Operator.MUST_PASS_ALL) {
+ filterListBase = new FilterListWithAND(filters);
+ } else if (operator == Operator.MUST_PASS_ONE) {
+ filterListBase = new FilterListWithOR(filters);
+ }
this.operator = operator;
- initPrevListForMustPassOne(rowFilters.size());
}
/**
* Constructor that takes a set of {@link Filter}s. The default operator MUST_PASS_ALL is assumed.
* All filters are cloned to internal list.
- * @param rowFilters list of filters
+ * @param filters list of filters
*/
- public FilterList(final List<Filter> rowFilters) {
- this(Operator.MUST_PASS_ALL, rowFilters);
+ public FilterList(final List<Filter> filters) {
+ this(Operator.MUST_PASS_ALL, filters);
}
/**
* Constructor that takes a var arg number of {@link Filter}s. The default operator MUST_PASS_ALL
* is assumed.
- * @param rowFilters
+ * @param filters
*/
- public FilterList(final Filter... rowFilters) {
- this(Operator.MUST_PASS_ALL, Arrays.asList(rowFilters));
+ public FilterList(final Filter... filters) {
+ this(Operator.MUST_PASS_ALL, Arrays.asList(filters));
}
/**
@@ -130,21 +105,10 @@ final public class FilterList extends FilterBase {
/**
* Constructor that takes a var arg number of {@link Filter}s and an operator.
* @param operator Operator to process filter set with.
- * @param rowFilters Filters to use
+ * @param filters Filters to use
*/
- public FilterList(final Operator operator, final Filter... rowFilters) {
- this(operator, Arrays.asList(rowFilters));
- }
-
- private void initPrevListForMustPassOne(int size) {
- if (operator == Operator.MUST_PASS_ONE) {
- if (this.prevFilterRCList == null) {
- prevFilterRCList = new ArrayList<>(Collections.nCopies(size, null));
- }
- if (this.prevCellList == null) {
- prevCellList = new ArrayList<>(Collections.nCopies(size, null));
- }
- }
+ public FilterList(final Operator operator, final Filter... filters) {
+ this(operator, Arrays.asList(filters));
}
/**
@@ -160,38 +124,15 @@ final public class FilterList extends FilterBase {
* @return filters
*/
public List<Filter> getFilters() {
- return filters;
+ return filterListBase.getFilters();
}
public int size() {
- return filters.size();
- }
-
- private boolean isEmpty() {
- return filters.isEmpty();
- }
-
- private static boolean checkAndGetReversed(List<Filter> rowFilters, boolean defaultValue) {
- if (rowFilters.isEmpty()) {
- return defaultValue;
- }
- Boolean retValue = rowFilters.get(0).isReversed();
- boolean allEqual = rowFilters.stream().map(Filter::isReversed).allMatch(retValue::equals);
- if (!allEqual) {
- throw new IllegalArgumentException("Filters in the list must have the same reversed flag");
- }
- return retValue;
+ return filterListBase.size();
}
public void addFilter(List<Filter> filters) {
- if (checkAndGetReversed(filters, isReversed()) != isReversed()) {
- throw new IllegalArgumentException("Filters in the list must have the same reversed flag");
- }
- this.filters.addAll(filters);
- if (operator == Operator.MUST_PASS_ONE) {
- this.prevFilterRCList.addAll(Collections.nCopies(filters.size(), null));
- this.prevCellList.addAll(Collections.nCopies(filters.size(), null));
- }
+ filterListBase.addFilterLists(filters);
}
/**
@@ -204,483 +145,51 @@ final public class FilterList extends FilterBase {
@Override
public void reset() throws IOException {
- for (int i = 0, n = filters.size(); i < n; i++) {
- filters.get(i).reset();
- if (operator == Operator.MUST_PASS_ONE) {
- prevFilterRCList.set(i, null);
- prevCellList.set(i, null);
- }
- }
- seekHintFilters.clear();
+ filterListBase.reset();
}
@Override
public boolean filterRowKey(byte[] rowKey, int offset, int length) throws IOException {
- if (isEmpty()) {
- return super.filterRowKey(rowKey, offset, length);
- }
- boolean flag = this.operator == Operator.MUST_PASS_ONE;
- for (int i = 0, n = filters.size(); i < n; i++) {
- Filter filter = filters.get(i);
- if (this.operator == Operator.MUST_PASS_ALL) {
- if (filter.filterAllRemaining() || filter.filterRowKey(rowKey, offset, length)) {
- flag = true;
- }
- } else if (this.operator == Operator.MUST_PASS_ONE) {
- if (!filter.filterAllRemaining() && !filter.filterRowKey(rowKey, offset, length)) {
- flag = false;
- }
- }
- }
- return flag;
+ return filterListBase.filterRowKey(rowKey, offset, length);
}
@Override
public boolean filterRowKey(Cell firstRowCell) throws IOException {
- if (isEmpty()) {
- return super.filterRowKey(firstRowCell);
- }
- boolean flag = this.operator == Operator.MUST_PASS_ONE;
- for (int i = 0, n = filters.size(); i < n; i++) {
- Filter filter = filters.get(i);
- if (this.operator == Operator.MUST_PASS_ALL) {
- if (filter.filterAllRemaining() || filter.filterRowKey(firstRowCell)) {
- flag = true;
- }
- } else if (this.operator == Operator.MUST_PASS_ONE) {
- if (!filter.filterAllRemaining() && !filter.filterRowKey(firstRowCell)) {
- flag = false;
- }
- }
- }
- return flag;
+ return filterListBase.filterRowKey(firstRowCell);
}
@Override
public boolean filterAllRemaining() throws IOException {
- if (isEmpty()) {
- return super.filterAllRemaining();
- }
- for (int i = 0, n = filters.size(); i < n; i++) {
- if (filters.get(i).filterAllRemaining()) {
- if (operator == Operator.MUST_PASS_ALL) {
- return true;
- }
- } else {
- if (operator == Operator.MUST_PASS_ONE) {
- return false;
- }
- }
- }
- return operator == Operator.MUST_PASS_ONE;
+ return filterListBase.filterAllRemaining();
}
@Override
public Cell transformCell(Cell c) throws IOException {
- if (isEmpty()) {
- return super.transformCell(c);
- }
- if (!CellUtil.equals(c, referenceCell)) {
- throw new IllegalStateException(
- "Reference Cell: " + this.referenceCell + " does not match: " + c);
- }
- // Copy transformedCell into a new cell and reset transformedCell & referenceCell to null for
- // Java GC optimization
- Cell cell = KeyValueUtil.copyToNewKeyValue(this.transformedCell);
- this.transformedCell = null;
- this.referenceCell = null;
- return cell;
- }
-
- /**
- * For MUST_PASS_ONE, we cannot make sure that when filter-A in filter list return NEXT_COL then
- * the next cell passing to filterList will be the first cell in next column, because if filter-B
- * in filter list return SKIP, then the filter list will return SKIP. In this case, we should pass
- * the cell following the previous cell, and it's possible that the next cell has the same column
- * as the previous cell even if filter-A has NEXT_COL returned for the previous cell. So we should
- * save the previous cell and the return code list when checking previous cell for every filter in
- * filter list, and verify if currentCell fit the previous return code, if fit then pass the currentCell
- * to the corresponding filter. (HBASE-17678)
- */
- private boolean shouldPassCurrentCellToFilter(Cell prevCell, Cell currentCell, int filterIdx)
- throws IOException {
- ReturnCode prevCode = this.prevFilterRCList.get(filterIdx);
- if (prevCell == null || prevCode == null) {
- return true;
- }
- switch (prevCode) {
- case INCLUDE:
- case SKIP:
- return true;
- case SEEK_NEXT_USING_HINT:
- Cell nextHintCell = getNextCellHint(prevCell);
- return nextHintCell == null
- || CellComparatorImpl.COMPARATOR.compare(currentCell, nextHintCell) >= 0;
- case NEXT_COL:
- case INCLUDE_AND_NEXT_COL:
- return !CellUtil.matchingRowColumn(prevCell, currentCell);
- case NEXT_ROW:
- case INCLUDE_AND_SEEK_NEXT_ROW:
- return !CellUtil.matchingRows(prevCell, currentCell);
- default:
- throw new IllegalStateException("Received code is not valid.");
- }
- }
-
- /**
- * FilterList with MUST_PASS_ALL choose the maximal forward step among sub-filters in filter list.
- * Let's call it: The Maximal Step Rule. So if filter-A in filter list return INCLUDE and filter-B
- * in filter list return INCLUDE_AND_NEXT_COL, then the filter list should return
- * INCLUDE_AND_NEXT_COL. For SEEK_NEXT_USING_HINT, it's more special, and in method
- * filterKeyValueWithMustPassAll(), if any sub-filter return SEEK_NEXT_USING_HINT, then our filter
- * list will return SEEK_NEXT_USING_HINT. so we don't care about the SEEK_NEXT_USING_HINT here. <br/>
- * <br/>
- * The jump step will be:
- *
- * <pre>
- * INCLUDE < SKIP < INCLUDE_AND_NEXT_COL < NEXT_COL < INCLUDE_AND_SEEK_NEXT_ROW < NEXT_ROW < SEEK_NEXT_USING_HINT
- * </pre>
- *
- * Here, we have the following map to describe The Maximal Step Rule. if current return code (for
- * previous sub-filters in filter list) is <strong>ReturnCode</strong>, and current filter returns
- * <strong>localRC</strong>, then we should return map[ReturnCode][localRC] for the merged result,
- * according to The Maximal Step Rule. <br/>
- *
- * <pre>
- * LocalCode\ReturnCode INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW SKIP NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
- * INCLUDE INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW SKIP NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
- * INCLUDE_AND_NEXT_COL INCLUDE_AND_NEXT_COL INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW NEXT_COL NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
- * INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW NEXT_ROW NEXT_ROW NEXT_ROW SEEK_NEXT_USING_HINT
- * SKIP SKIP NEXT_COL NEXT_ROW SKIP NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
- * NEXT_COL NEXT_COL NEXT_COL NEXT_ROW NEXT_COL NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
- * NEXT_ROW NEXT_ROW NEXT_ROW NEXT_ROW NEXT_ROW NEXT_ROW NEXT_ROW SEEK_NEXT_USING_HINT
- * SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT
- * </pre>
- * @param rc Return code which is calculated by previous sub-filter(s) in filter list.
- * @param localRC Return code of the current sub-filter in filter list.
- * @return Return code which is merged by the return code of previous sub-filter(s) and the return
- * code of current sub-filter.
- */
- private ReturnCode mergeReturnCodeForAndOperator(ReturnCode rc, ReturnCode localRC) {
- if (rc == ReturnCode.SEEK_NEXT_USING_HINT || localRC == ReturnCode.SEEK_NEXT_USING_HINT) {
- return ReturnCode.SEEK_NEXT_USING_HINT;
- }
- switch (localRC) {
- case INCLUDE:
- return rc;
- case INCLUDE_AND_NEXT_COL:
- if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL)) {
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- }
- if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
- return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
- }
- if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL)) {
- return ReturnCode.NEXT_COL;
- }
- if (isInReturnCodes(rc, ReturnCode.NEXT_ROW)) {
- return ReturnCode.NEXT_ROW;
- }
- break;
- case INCLUDE_AND_SEEK_NEXT_ROW:
- if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
- ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
- return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
- }
- if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) {
- return ReturnCode.NEXT_ROW;
- }
- break;
- case SKIP:
- if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP)) {
- return ReturnCode.SKIP;
- }
- if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.NEXT_COL)) {
- return ReturnCode.NEXT_COL;
- }
- if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) {
- return ReturnCode.NEXT_ROW;
- }
- break;
- case NEXT_COL:
- if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.SKIP,
- ReturnCode.NEXT_COL)) {
- return ReturnCode.NEXT_COL;
- }
- if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) {
- return ReturnCode.NEXT_ROW;
- }
- break;
- case NEXT_ROW:
- return ReturnCode.NEXT_ROW;
- }
- throw new IllegalStateException("Received code is not valid. rc: " + rc + ", localRC: "
- + localRC);
- }
-
- private ReturnCode filterKeyValueWithMustPassAll(Cell c) throws IOException {
- ReturnCode rc = ReturnCode.INCLUDE;
- Cell transformed = c;
- this.seekHintFilters.clear();
- for (int i = 0, n = filters.size(); i < n; i++) {
- Filter filter = filters.get(i);
- if (filter.filterAllRemaining()) {
- return ReturnCode.NEXT_ROW;
- }
- ReturnCode localRC = filter.filterKeyValue(c);
- rc = mergeReturnCodeForAndOperator(rc, localRC);
-
- // For INCLUDE* case, we need to update the transformed cell.
- if (isInReturnCodes(localRC, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
- ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
- transformed = filter.transformCell(transformed);
- }
- if (localRC == ReturnCode.SEEK_NEXT_USING_HINT) {
- seekHintFilters.add(filter);
- }
- }
- this.transformedCell = transformed;
- if (!seekHintFilters.isEmpty()) {
- return ReturnCode.SEEK_NEXT_USING_HINT;
- }
- return rc;
- }
-
- private void updatePrevFilterRCList(int index, ReturnCode currentRC) {
- prevFilterRCList.set(index, currentRC);
- }
-
- private void updatePrevCellList(int index, Cell currentCell, ReturnCode currentRC) {
- if (currentCell == null || currentRC == ReturnCode.INCLUDE || currentRC == ReturnCode.SKIP) {
- // If previous return code is INCLUDE or SKIP, we should always pass the next cell to the
- // corresponding sub-filter(need not test shouldPassCurrentCellToFilter() method), So we
- // need not save current cell to prevCellList for saving heap memory.
- prevCellList.set(index, null);
- } else {
- prevCellList.set(index, KeyValueUtil.toNewKeyCell(currentCell));
- }
- }
-
- private static boolean isInReturnCodes(ReturnCode testRC, ReturnCode... returnCodes) {
- return Arrays.stream(returnCodes).anyMatch(testRC::equals);
- }
-
- /**
- * FilterList with MUST_PASS_ONE choose the minimal forward step among sub-filter in filter list.
- * Let's call it: The Minimal Step Rule. So if filter-A in filter list return INCLUDE and filter-B
- * in filter list return INCLUDE_AND_NEXT_COL, then the filter list should return INCLUDE. For
- * SEEK_NEXT_USING_HINT, it's more special, because we do not know how far it will forward, so we
- * use SKIP by default.<br/>
- * <br/>
- * The jump step will be:
- *
- * <pre>
- * INCLUDE < SKIP < INCLUDE_AND_NEXT_COL < NEXT_COL < INCLUDE_AND_SEEK_NEXT_ROW < NEXT_ROW < SEEK_NEXT_USING_HINT
- * </pre>
- *
- * Here, we have the following map to describe The Minimal Step Rule. if current return code (for
- * previous sub-filters in filter list) is <strong>ReturnCode</strong>, and current filter returns
- * <strong>localRC</strong>, then we should return map[ReturnCode][localRC] for the merged result,
- * according to The Minimal Step Rule.<br/>
- *
- * <pre>
- * LocalCode\ReturnCode INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW SKIP NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
- * INCLUDE INCLUDE INCLUDE INCLUDE INCLUDE INCLUDE INCLUDE INCLUDE
- * INCLUDE_AND_NEXT_COL INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_NEXT_COL INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_NEXT_COL INCLUDE
- * INCLUDE_AND_SEEK_NEXT_ROW INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW INCLUDE
- * SKIP INCLUDE INCLUDE INCLUDE SKIP SKIP SKIP SKIP
- * NEXT_COL INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_NEXT_COL SKIP NEXT_COL NEXT_COL SKIP
- * NEXT_ROW INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW SKIP NEXT_COL NEXT_ROW SKIP
- * SEEK_NEXT_USING_HINT INCLUDE INCLUDE INCLUDE SKIP SKIP SKIP SEEK_NEXT_USING_HINT
- * </pre>
- * @param rc Return code which is calculated by previous sub-filter(s) in filter list.
- * @param localRC Return code of the current sub-filter in filter list.
- * @return Return code which is merged by the return code of previous sub-filter(s) and the return
- * code of current sub-filter.
- */
- private ReturnCode mergeReturnCodeForOrOperator(ReturnCode rc, ReturnCode localRC) {
- if (rc == null) return localRC;
- switch (localRC) {
- case INCLUDE:
- return ReturnCode.INCLUDE;
- case INCLUDE_AND_NEXT_COL:
- if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP,
- ReturnCode.SEEK_NEXT_USING_HINT)) {
- return ReturnCode.INCLUDE;
- }
- if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW,
- ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) {
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- }
- break;
- case INCLUDE_AND_SEEK_NEXT_ROW:
- if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP,
- ReturnCode.SEEK_NEXT_USING_HINT)) {
- return ReturnCode.INCLUDE;
- }
- if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.NEXT_COL)) {
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- }
- if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) {
- return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
- }
- break;
- case SKIP:
- if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
- ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
- return ReturnCode.INCLUDE;
- }
- if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW,
- ReturnCode.SEEK_NEXT_USING_HINT)) {
- return ReturnCode.SKIP;
- }
- break;
- case NEXT_COL:
- if (isInReturnCodes(rc, ReturnCode.INCLUDE)) {
- return ReturnCode.INCLUDE;
- }
- if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL,
- ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- }
- if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.SEEK_NEXT_USING_HINT)) {
- return ReturnCode.SKIP;
- }
- break;
- case NEXT_ROW:
- if (isInReturnCodes(rc, ReturnCode.INCLUDE)) {
- return ReturnCode.INCLUDE;
- }
- if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL)) {
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- }
- if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
- return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
- }
- if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.SEEK_NEXT_USING_HINT)) {
- return ReturnCode.SKIP;
- }
- if (isInReturnCodes(rc, ReturnCode.NEXT_COL)) {
- return ReturnCode.NEXT_COL;
- }
- if (isInReturnCodes(rc, ReturnCode.NEXT_ROW)) {
- return ReturnCode.NEXT_ROW;
- }
- break;
- case SEEK_NEXT_USING_HINT:
- if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
- ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
- return ReturnCode.INCLUDE;
- }
- if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) {
- return ReturnCode.SKIP;
- }
- if (isInReturnCodes(rc, ReturnCode.SEEK_NEXT_USING_HINT)) {
- return ReturnCode.SEEK_NEXT_USING_HINT;
- }
- break;
- }
- throw new IllegalStateException(
- "Received code is not valid. rc: " + rc + ", localRC: " + localRC);
- }
-
- private ReturnCode filterKeyValueWithMustPassOne(Cell c) throws IOException {
- ReturnCode rc = null;
- boolean everyFilterReturnHint = true;
- Cell transformed = c;
- for (int i = 0, n = filters.size(); i < n; i++) {
- Filter filter = filters.get(i);
- Cell prevCell = this.prevCellList.get(i);
- if (filter.filterAllRemaining() || !shouldPassCurrentCellToFilter(prevCell, c, i)) {
- everyFilterReturnHint = false;
- continue;
- }
-
- ReturnCode localRC = filter.filterKeyValue(c);
-
- // Update previous return code and previous cell for filter[i].
- updatePrevFilterRCList(i, localRC);
- updatePrevCellList(i, c, localRC);
-
- if (localRC != ReturnCode.SEEK_NEXT_USING_HINT) {
- everyFilterReturnHint = false;
- }
-
- rc = mergeReturnCodeForOrOperator(rc, localRC);
-
- // For INCLUDE* case, we need to update the transformed cell.
- if (isInReturnCodes(localRC, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
- ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
- transformed = filter.transformCell(transformed);
- }
- }
-
- this.transformedCell = transformed;
- if (everyFilterReturnHint) {
- return ReturnCode.SEEK_NEXT_USING_HINT;
- } else if (rc == null) {
- // Each sub-filter in filter list got true for filterAllRemaining().
- return ReturnCode.SKIP;
- } else {
- return rc;
- }
+ return filterListBase.transformCell(c);
}
@Override
public ReturnCode filterKeyValue(Cell c) throws IOException {
- if (isEmpty()) {
- return ReturnCode.INCLUDE;
- }
- this.referenceCell = c;
-
- if (operator == Operator.MUST_PASS_ALL) {
- return filterKeyValueWithMustPassAll(c);
- } else {
- return filterKeyValueWithMustPassOne(c);
- }
+ return filterListBase.filterKeyValue(c);
}
/**
- * Filters that never filter by modifying the returned List of Cells can
- * inherit this implementation that does nothing.
- *
- * {@inheritDoc}
+ * Filters that never filter by modifying the returned List of Cells can inherit this
+ * implementation that does nothing. {@inheritDoc}
*/
@Override
public void filterRowCells(List<Cell> cells) throws IOException {
- for (int i = 0, n = filters.size(); i < n; i++) {
- filters.get(i).filterRowCells(cells);
- }
+ filterListBase.filterRowCells(cells);
}
@Override
public boolean hasFilterRow() {
- for (int i = 0, n = filters.size(); i < n; i++) {
- if (filters.get(i).hasFilterRow()) {
- return true;
- }
- }
- return false;
+ return filterListBase.hasFilterRow();
}
@Override
public boolean filterRow() throws IOException {
- if (isEmpty()) {
- return super.filterRow();
- }
- for (int i = 0, n = filters.size(); i < n; i++) {
- Filter filter = filters.get(i);
- if (operator == Operator.MUST_PASS_ALL) {
- if (filter.filterRow()) {
- return true;
- }
- } else if (operator == Operator.MUST_PASS_ONE) {
- if (!filter.filterRow()) {
- return false;
- }
- }
- }
- return operator == Operator.MUST_PASS_ONE;
+ return filterListBase.filterRow();
}
/**
@@ -689,6 +198,7 @@ final public class FilterList extends FilterBase {
public byte[] toByteArray() throws IOException {
FilterProtos.FilterList.Builder builder = FilterProtos.FilterList.newBuilder();
builder.setOperator(FilterProtos.FilterList.Operator.valueOf(operator.name()));
+ ArrayList<Filter> filters = filterListBase.getFilters();
for (int i = 0, n = filters.size(); i < n; i++) {
builder.addFilters(ProtobufUtil.toFilter(filters.get(i)));
}
@@ -701,8 +211,7 @@ final public class FilterList extends FilterBase {
* @throws DeserializationException
* @see #toByteArray
*/
- public static FilterList parseFrom(final byte [] pbBytes)
- throws DeserializationException {
+ public static FilterList parseFrom(final byte[] pbBytes) throws DeserializationException {
FilterProtos.FilterList proto;
try {
proto = FilterProtos.FilterList.parseFrom(pbBytes);
@@ -724,96 +233,42 @@ final public class FilterList extends FilterBase {
/**
* @param other
- * @return true if and only if the fields of the filter that are serialized
- * are equal to the corresponding fields in other. Used for testing.
+ * @return true if and only if the fields of the filter that are serialized are equal to the
+ * corresponding fields in other. Used for testing.
*/
boolean areSerializedFieldsEqual(Filter other) {
if (other == this) return true;
if (!(other instanceof FilterList)) return false;
- FilterList o = (FilterList)other;
- return this.getOperator().equals(o.getOperator()) &&
- ((this.getFilters() == o.getFilters())
- || this.getFilters().equals(o.getFilters()));
+ FilterList o = (FilterList) other;
+ return this.getOperator().equals(o.getOperator())
+ && ((this.getFilters() == o.getFilters()) || this.getFilters().equals(o.getFilters()));
}
@Override
public Cell getNextCellHint(Cell currentCell) throws IOException {
- if (isEmpty()) {
- return super.getNextCellHint(currentCell);
- }
- Cell keyHint = null;
- if (operator == Operator.MUST_PASS_ALL) {
- for (Filter filter : seekHintFilters) {
- if (filter.filterAllRemaining()) continue;
- Cell curKeyHint = filter.getNextCellHint(currentCell);
- if (keyHint == null) {
- keyHint = curKeyHint;
- continue;
- }
- if (CellComparatorImpl.COMPARATOR.compare(keyHint, curKeyHint) < 0) {
- keyHint = curKeyHint;
- }
- }
- return keyHint;
- }
-
- // If any condition can pass, we need to keep the min hint
- for (int i = 0, n = filters.size(); i < n; i++) {
- if (filters.get(i).filterAllRemaining()) {
- continue;
- }
- Cell curKeyHint = filters.get(i).getNextCellHint(currentCell);
- if (curKeyHint == null) {
- // If we ever don't have a hint and this is must-pass-one, then no hint
- return null;
- }
- // If this is the first hint we find, set it
- if (keyHint == null) {
- keyHint = curKeyHint;
- continue;
- }
- if (CellComparatorImpl.COMPARATOR.compare(keyHint, curKeyHint) > 0) {
- keyHint = curKeyHint;
- }
- }
- return keyHint;
+ return this.filterListBase.getNextCellHint(currentCell);
}
@Override
public boolean isFamilyEssential(byte[] name) throws IOException {
- if (isEmpty()) {
- return super.isFamilyEssential(name);
- }
- for (int i = 0, n = filters.size(); i < n; i++) {
- if (filters.get(i).isFamilyEssential(name)) {
- return true;
- }
- }
- return false;
+ return this.filterListBase.isFamilyEssential(name);
}
@Override
public void setReversed(boolean reversed) {
- for (int i = 0, n = filters.size(); i < n; i++) {
- filters.get(i).setReversed(reversed);
- }
this.reversed = reversed;
+ this.filterListBase.setReversed(reversed);
}
@Override
- public String toString() {
- return toString(MAX_LOG_FILTERS);
+ public boolean isReversed() {
+ assert this.reversed == this.filterListBase.isReversed();
+ return this.reversed;
}
- protected String toString(int maxFilters) {
- int endIndex = this.filters.size() < maxFilters
- ? this.filters.size() : maxFilters;
- return String.format("%s %s (%d/%d): %s",
- this.getClass().getSimpleName(),
- this.operator == Operator.MUST_PASS_ALL ? "AND" : "OR",
- endIndex,
- this.filters.size(),
- this.filters.subList(0, endIndex).toString());
+ @Override
+ public String toString() {
+ return this.filterListBase.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/59275100/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java
new file mode 100644
index 0000000..7fa0245
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java
@@ -0,0 +1,159 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellComparatorImpl;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Base class for FilterList. Currently, we have two sub-classes to extend this class:
+ * {@link FilterListWithOR}, {@link FilterListWithAND}.
+ */
+@InterfaceAudience.Private
+public abstract class FilterListBase extends FilterBase {
+ private static final int MAX_LOG_FILTERS = 5;
+ protected final ArrayList<Filter> filters;
+
+ /** Reference Cell used by {@link #transformCell(Cell)} for validation purpose. */
+ protected Cell referenceCell = null;
+
+ /**
+ * When filtering a given Cell in {@link #filterKeyValue(Cell)}, this stores the transformed Cell
+ * to be returned by {@link #transformCell(Cell)}. Individual filters transformation are applied
+ * only when the filter includes the Cell. Transformations are composed in the order specified by
+ * {@link #filters}.
+ */
+ protected Cell transformedCell = null;
+
+ public FilterListBase(List<Filter> filters) {
+ reversed = checkAndGetReversed(filters, reversed);
+ this.filters = new ArrayList<>(filters);
+ }
+
+ protected static boolean isInReturnCodes(ReturnCode testRC, ReturnCode... returnCodes) {
+ return Arrays.stream(returnCodes).anyMatch(testRC::equals);
+ }
+
+ protected static boolean checkAndGetReversed(List<Filter> rowFilters, boolean defaultValue) {
+ if (rowFilters.isEmpty()) {
+ return defaultValue;
+ }
+ Boolean retValue = rowFilters.get(0).isReversed();
+ boolean allEqual = rowFilters.stream().map(Filter::isReversed).allMatch(retValue::equals);
+ if (!allEqual) {
+ throw new IllegalArgumentException("Filters in the list must have the same reversed flag");
+ }
+ return retValue;
+ }
+
+ public abstract void addFilterLists(List<Filter> filters);
+
+ public int size() {
+ return this.filters.size();
+ }
+
+ public boolean isEmpty() {
+ return this.filters.isEmpty();
+ }
+
+ public ArrayList<Filter> getFilters() {
+ return this.filters;
+ }
+
+ protected int compareCell(Cell a, Cell b) {
+ int cmp = CellComparatorImpl.COMPARATOR.compare(a, b);
+ return reversed ? -1 * cmp : cmp;
+ }
+
+ @Override
+ public Cell transformCell(Cell c) throws IOException {
+ if (isEmpty()) {
+ return super.transformCell(c);
+ }
+ if (!CellUtil.equals(c, referenceCell)) {
+ throw new IllegalStateException(
+ "Reference Cell: " + this.referenceCell + " does not match: " + c);
+ }
+ // Copy transformedCell into a new cell and reset transformedCell & referenceCell to null for
+ // Java GC optimization
+ Cell cell = KeyValueUtil.copyToNewKeyValue(this.transformedCell);
+ this.transformedCell = null;
+ this.referenceCell = null;
+ return cell;
+ }
+
+ /**
+ * Filters that never filter by modifying the returned List of Cells can inherit this
+ * implementation that does nothing. {@inheritDoc}
+ */
+ @Override
+ public void filterRowCells(List<Cell> cells) throws IOException {
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ filters.get(i).filterRowCells(cells);
+ }
+ }
+
+ @Override
+ public boolean hasFilterRow() {
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ if (filters.get(i).hasFilterRow()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean isFamilyEssential(byte[] name) throws IOException {
+ if (this.filters.isEmpty()) {
+ return super.isFamilyEssential(name);
+ }
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ if (filters.get(i).isFamilyEssential(name)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public void setReversed(boolean reversed) {
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ filters.get(i).setReversed(reversed);
+ }
+ this.reversed = reversed;
+ }
+
+ @Override
+ public String toString() {
+ int endIndex = this.size() < MAX_LOG_FILTERS ? this.size() : MAX_LOG_FILTERS;
+ return formatLogFilters(filters.subList(0, endIndex));
+ }
+
+ protected abstract String formatLogFilters(List<Filter> logFilters);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/59275100/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java
new file mode 100644
index 0000000..fa979c0
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java
@@ -0,0 +1,273 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * FilterListWithAND represents an ordered list of filters which will be evaluated with an AND
+ * operator.
+ */
+@InterfaceAudience.Private
+public class FilterListWithAND extends FilterListBase {
+
+ private Set<Filter> seekHintFilter = new HashSet<>();
+
+ public FilterListWithAND(List<Filter> filters) {
+ super(filters);
+ }
+
+ @Override
+ public void addFilterLists(List<Filter> filters) {
+ if (checkAndGetReversed(filters, isReversed()) != isReversed()) {
+ throw new IllegalArgumentException("Filters in the list must have the same reversed flag");
+ }
+ this.filters.addAll(filters);
+ }
+
+ @Override
+ protected String formatLogFilters(List<Filter> logFilters) {
+ return String.format("FilterList AND (%d/%d): %s", logFilters.size(), this.size(),
+ logFilters.toString());
+ }
+
+ /**
+ * FilterList with MUST_PASS_ALL choose the maximal forward step among sub-filters in filter list.
+ * Let's call it: The Maximal Step Rule. So if filter-A in filter list return INCLUDE and filter-B
+ * in filter list return INCLUDE_AND_NEXT_COL, then the filter list should return
+ * INCLUDE_AND_NEXT_COL. For SEEK_NEXT_USING_HINT, it's more special, and in method
+ * filterKeyValueWithMustPassAll(), if any sub-filter return SEEK_NEXT_USING_HINT, then our filter
+ * list will return SEEK_NEXT_USING_HINT. so we don't care about the SEEK_NEXT_USING_HINT here.
+ * <br/>
+ * <br/>
+ * The jump step will be:
+ *
+ * <pre>
+ * INCLUDE < SKIP < INCLUDE_AND_NEXT_COL < NEXT_COL < INCLUDE_AND_SEEK_NEXT_ROW < NEXT_ROW < SEEK_NEXT_USING_HINT
+ * </pre>
+ *
+ * Here, we have the following map to describe The Maximal Step Rule. if current return code (for
+ * previous sub-filters in filter list) is <strong>ReturnCode</strong>, and current filter returns
+ * <strong>localRC</strong>, then we should return map[ReturnCode][localRC] for the merged result,
+ * according to The Maximal Step Rule. <br/>
+ *
+ * <pre>
+ * LocalCode\ReturnCode INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW SKIP NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
+ * INCLUDE INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW SKIP NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
+ * INCLUDE_AND_NEXT_COL INCLUDE_AND_NEXT_COL INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW NEXT_COL NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
+ * INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW NEXT_ROW NEXT_ROW NEXT_ROW SEEK_NEXT_USING_HINT
+ * SKIP SKIP NEXT_COL NEXT_ROW SKIP NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
+ * NEXT_COL NEXT_COL NEXT_COL NEXT_ROW NEXT_COL NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
+ * NEXT_ROW NEXT_ROW NEXT_ROW NEXT_ROW NEXT_ROW NEXT_ROW NEXT_ROW SEEK_NEXT_USING_HINT
+ * SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT SEEK_NEXT_USING_HINT
+ * </pre>
+ *
+ * @param rc Return code which is calculated by previous sub-filter(s) in filter list.
+ * @param localRC Return code of the current sub-filter in filter list.
+ * @return Return code which is merged by the return code of previous sub-filter(s) and the return
+ * code of current sub-filter.
+ */
+ private ReturnCode mergeReturnCode(ReturnCode rc, ReturnCode localRC) {
+ if (rc == ReturnCode.SEEK_NEXT_USING_HINT || localRC == ReturnCode.SEEK_NEXT_USING_HINT) {
+ return ReturnCode.SEEK_NEXT_USING_HINT;
+ }
+ switch (localRC) {
+ case INCLUDE:
+ return rc;
+ case INCLUDE_AND_NEXT_COL:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL)) {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL)) {
+ return ReturnCode.NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.NEXT_ROW;
+ }
+ break;
+ case INCLUDE_AND_SEEK_NEXT_ROW:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
+ ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.NEXT_ROW;
+ }
+ break;
+ case SKIP:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP)) {
+ return ReturnCode.SKIP;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.NEXT_COL)) {
+ return ReturnCode.NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.NEXT_ROW;
+ }
+ break;
+ case NEXT_COL:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.SKIP,
+ ReturnCode.NEXT_COL)) {
+ return ReturnCode.NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.NEXT_ROW;
+ }
+ break;
+ case NEXT_ROW:
+ return ReturnCode.NEXT_ROW;
+ }
+ throw new IllegalStateException(
+ "Received code is not valid. rc: " + rc + ", localRC: " + localRC);
+ }
+
+ private ReturnCode filterKeyValueWithMustPassAll(Cell c) throws IOException {
+ ReturnCode rc = ReturnCode.INCLUDE;
+ Cell transformed = c;
+ this.seekHintFilter.clear();
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ Filter filter = filters.get(i);
+ if (filter.filterAllRemaining()) {
+ return ReturnCode.NEXT_ROW;
+ }
+ ReturnCode localRC = filter.filterKeyValue(c);
+ rc = mergeReturnCode(rc, localRC);
+
+ // For INCLUDE* case, we need to update the transformed cell.
+ if (isInReturnCodes(localRC, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
+ ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ transformed = filter.transformCell(transformed);
+ }
+ if (localRC == ReturnCode.SEEK_NEXT_USING_HINT) {
+ seekHintFilter.add(filter);
+ }
+ }
+ this.transformedCell = transformed;
+ if (!seekHintFilter.isEmpty()) {
+ return ReturnCode.SEEK_NEXT_USING_HINT;
+ }
+ return rc;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell c) throws IOException {
+ if (isEmpty()) {
+ return ReturnCode.INCLUDE;
+ }
+ this.referenceCell = c;
+ return filterKeyValueWithMustPassAll(c);
+ }
+
+ @Override
+ public void reset() throws IOException {
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ filters.get(i).reset();
+ }
+ seekHintFilter.clear();
+ }
+
+ @Override
+ public boolean filterRowKey(byte[] rowKey, int offset, int length) throws IOException {
+ if (isEmpty()) {
+ return super.filterRowKey(rowKey, offset, length);
+ }
+ boolean retVal = false;
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ Filter filter = filters.get(i);
+ if (filter.filterAllRemaining() || filter.filterRowKey(rowKey, offset, length)) {
+ retVal = true;
+ }
+ }
+ return retVal;
+ }
+
+ @Override
+ public boolean filterRowKey(Cell firstRowCell) throws IOException {
+ if (isEmpty()) {
+ return super.filterRowKey(firstRowCell);
+ }
+ boolean retVal = false;
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ Filter filter = filters.get(i);
+ if (filter.filterAllRemaining() || filter.filterRowKey(firstRowCell)) {
+ retVal = true;
+ }
+ }
+ return retVal;
+ }
+
+ @Override
+ public boolean filterAllRemaining() throws IOException {
+ if (isEmpty()) {
+ return super.filterAllRemaining();
+ }
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ if (filters.get(i).filterAllRemaining()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean filterRow() throws IOException {
+ if (isEmpty()) {
+ return super.filterRow();
+ }
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ Filter filter = filters.get(i);
+ if (filter.filterRow()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public Cell getNextCellHint(Cell currentCell) throws IOException {
+ if (isEmpty()) {
+ return super.getNextCellHint(currentCell);
+ }
+ Cell maxHint = null;
+ for (Filter filter : seekHintFilter) {
+ if (filter.filterAllRemaining()) {
+ continue;
+ }
+ Cell curKeyHint = filter.getNextCellHint(currentCell);
+ if (maxHint == null) {
+ maxHint = curKeyHint;
+ continue;
+ }
+ if (this.compareCell(maxHint, curKeyHint) < 0) {
+ maxHint = curKeyHint;
+ }
+ }
+ return maxHint;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/59275100/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java
new file mode 100644
index 0000000..55306f2
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java
@@ -0,0 +1,383 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * FilterListWithOR represents an ordered list of filters which will be evaluated with an OR
+ * operator.
+ */
+@InterfaceAudience.Private
+public class FilterListWithOR extends FilterListBase {
+
+ /**
+ * Save previous return code and previous cell for every filter in filter list. For MUST_PASS_ONE,
+ * we use the previous return code to decide whether we should pass current cell encountered to
+ * the filter. For MUST_PASS_ALL, the two list are meaningless.
+ */
+ private List<ReturnCode> prevFilterRCList = null;
+ private List<Cell> prevCellList = null;
+
+ public FilterListWithOR(List<Filter> filters) {
+ super(filters);
+ prevFilterRCList = new ArrayList<>(Collections.nCopies(filters.size(), null));
+ prevCellList = new ArrayList<>(Collections.nCopies(filters.size(), null));
+ }
+
+ @Override
+ public void addFilterLists(List<Filter> filters) {
+ if (checkAndGetReversed(filters, isReversed()) != isReversed()) {
+ throw new IllegalArgumentException("Filters in the list must have the same reversed flag");
+ }
+ this.filters.addAll(filters);
+ this.prevFilterRCList.addAll(Collections.nCopies(filters.size(), null));
+ this.prevCellList.addAll(Collections.nCopies(filters.size(), null));
+ }
+
+ @Override
+ protected String formatLogFilters(List<Filter> logFilters) {
+ return String.format("FilterList OR (%d/%d): %s", logFilters.size(), this.size(),
+ logFilters.toString());
+ }
+
+ /**
+ * For MUST_PASS_ONE, we cannot make sure that when filter-A in filter list return NEXT_COL then
+ * the next cell passing to filterList will be the first cell in next column, because if filter-B
+ * in filter list return SKIP, then the filter list will return SKIP. In this case, we should pass
+ * the cell following the previous cell, and it's possible that the next cell has the same column
+ * as the previous cell even if filter-A has NEXT_COL returned for the previous cell. So we should
+ * save the previous cell and the return code list when checking previous cell for every filter in
+ * filter list, and verify if currentCell fit the previous return code, if fit then pass the
+ * currentCell to the corresponding filter. (HBASE-17678)
+ */
+ private boolean shouldPassCurrentCellToFilter(Cell prevCell, Cell currentCell, int filterIdx)
+ throws IOException {
+ ReturnCode prevCode = this.prevFilterRCList.get(filterIdx);
+ if (prevCell == null || prevCode == null) {
+ return true;
+ }
+ switch (prevCode) {
+ case INCLUDE:
+ case SKIP:
+ return true;
+ case SEEK_NEXT_USING_HINT:
+ Cell nextHintCell = getNextCellHint(prevCell);
+ return nextHintCell == null || this.compareCell(currentCell, nextHintCell) >= 0;
+ case NEXT_COL:
+ case INCLUDE_AND_NEXT_COL:
+ return !CellUtil.matchingRowColumn(prevCell, currentCell);
+ case NEXT_ROW:
+ case INCLUDE_AND_SEEK_NEXT_ROW:
+ return !CellUtil.matchingRows(prevCell, currentCell);
+ default:
+ throw new IllegalStateException("Received code is not valid.");
+ }
+ }
+
+ /**
+ * FilterList with MUST_PASS_ONE choose the minimal forward step among sub-filter in filter list.
+ * Let's call it: The Minimal Step Rule. So if filter-A in filter list return INCLUDE and filter-B
+ * in filter list return INCLUDE_AND_NEXT_COL, then the filter list should return INCLUDE. For
+ * SEEK_NEXT_USING_HINT, it's more special, because we do not know how far it will forward, so we
+ * use SKIP by default.<br/>
+ * <br/>
+ * The jump step will be:
+ *
+ * <pre>
+ * INCLUDE < SKIP < INCLUDE_AND_NEXT_COL < NEXT_COL < INCLUDE_AND_SEEK_NEXT_ROW < NEXT_ROW < SEEK_NEXT_USING_HINT
+ * </pre>
+ *
+ * Here, we have the following map to describe The Minimal Step Rule. if current return code (for
+ * previous sub-filters in filter list) is <strong>ReturnCode</strong>, and current filter returns
+ * <strong>localRC</strong>, then we should return map[ReturnCode][localRC] for the merged result,
+ * according to The Minimal Step Rule.<br/>
+ *
+ * <pre>
+ * LocalCode\ReturnCode INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW SKIP NEXT_COL NEXT_ROW SEEK_NEXT_USING_HINT
+ * INCLUDE INCLUDE INCLUDE INCLUDE INCLUDE INCLUDE INCLUDE INCLUDE
+ * INCLUDE_AND_NEXT_COL INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_NEXT_COL INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_NEXT_COL INCLUDE
+ * INCLUDE_AND_SEEK_NEXT_ROW INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW INCLUDE
+ * SKIP INCLUDE INCLUDE INCLUDE SKIP SKIP SKIP SKIP
+ * NEXT_COL INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_NEXT_COL SKIP NEXT_COL NEXT_COL SKIP
+ * NEXT_ROW INCLUDE INCLUDE_AND_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW SKIP NEXT_COL NEXT_ROW SKIP
+ * SEEK_NEXT_USING_HINT INCLUDE INCLUDE INCLUDE SKIP SKIP SKIP SEEK_NEXT_USING_HINT
+ * </pre>
+ *
+ * @param rc Return code which is calculated by previous sub-filter(s) in filter list.
+ * @param localRC Return code of the current sub-filter in filter list.
+ * @return Return code which is merged by the return code of previous sub-filter(s) and the return
+ * code of current sub-filter.
+ */
+ private ReturnCode mergeReturnCode(ReturnCode rc, ReturnCode localRC) {
+ if (rc == null) return localRC;
+ switch (localRC) {
+ case INCLUDE:
+ return ReturnCode.INCLUDE;
+ case INCLUDE_AND_NEXT_COL:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP,
+ ReturnCode.SEEK_NEXT_USING_HINT)) {
+ return ReturnCode.INCLUDE;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW,
+ ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ break;
+ case INCLUDE_AND_SEEK_NEXT_ROW:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP,
+ ReturnCode.SEEK_NEXT_USING_HINT)) {
+ return ReturnCode.INCLUDE;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.NEXT_COL)) {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
+ }
+ break;
+ case SKIP:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
+ ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ return ReturnCode.INCLUDE;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW,
+ ReturnCode.SEEK_NEXT_USING_HINT)) {
+ return ReturnCode.SKIP;
+ }
+ break;
+ case NEXT_COL:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE)) {
+ return ReturnCode.INCLUDE;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL,
+ ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.SEEK_NEXT_USING_HINT)) {
+ return ReturnCode.SKIP;
+ }
+ break;
+ case NEXT_ROW:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE)) {
+ return ReturnCode.INCLUDE;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL)) {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.SEEK_NEXT_USING_HINT)) {
+ return ReturnCode.SKIP;
+ }
+ if (isInReturnCodes(rc, ReturnCode.NEXT_COL)) {
+ return ReturnCode.NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.NEXT_ROW;
+ }
+ break;
+ case SEEK_NEXT_USING_HINT:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
+ ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ return ReturnCode.INCLUDE;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.SKIP;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SEEK_NEXT_USING_HINT)) {
+ return ReturnCode.SEEK_NEXT_USING_HINT;
+ }
+ break;
+ }
+ throw new IllegalStateException(
+ "Received code is not valid. rc: " + rc + ", localRC: " + localRC);
+ }
+
+ private void updatePrevFilterRCList(int index, ReturnCode currentRC) {
+ prevFilterRCList.set(index, currentRC);
+ }
+
+ private void updatePrevCellList(int index, Cell currentCell, ReturnCode currentRC) {
+ if (currentCell == null || currentRC == ReturnCode.INCLUDE || currentRC == ReturnCode.SKIP) {
+ // If previous return code is INCLUDE or SKIP, we should always pass the next cell to the
+ // corresponding sub-filter(need not test shouldPassCurrentCellToFilter() method), So we
+ // need not save current cell to prevCellList for saving heap memory.
+ prevCellList.set(index, null);
+ } else {
+ prevCellList.set(index, KeyValueUtil.toNewKeyCell(currentCell));
+ }
+ }
+
+ private ReturnCode filterKeyValueWithMustPassOne(Cell c) throws IOException {
+ ReturnCode rc = null;
+ boolean everyFilterReturnHint = true;
+ Cell transformed = c;
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ Filter filter = filters.get(i);
+
+ Cell prevCell = this.prevCellList.get(i);
+ if (filter.filterAllRemaining() || !shouldPassCurrentCellToFilter(prevCell, c, i)) {
+ everyFilterReturnHint = false;
+ continue;
+ }
+
+ ReturnCode localRC = filter.filterKeyValue(c);
+
+ // Update previous return code and previous cell for filter[i].
+ updatePrevFilterRCList(i, localRC);
+ updatePrevCellList(i, c, localRC);
+
+ if (localRC != ReturnCode.SEEK_NEXT_USING_HINT) {
+ everyFilterReturnHint = false;
+ }
+
+ rc = mergeReturnCode(rc, localRC);
+
+ // For INCLUDE* case, we need to update the transformed cell.
+ if (isInReturnCodes(localRC, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
+ ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ transformed = filter.transformCell(transformed);
+ }
+ }
+
+ this.transformedCell = transformed;
+ if (everyFilterReturnHint) {
+ return ReturnCode.SEEK_NEXT_USING_HINT;
+ } else if (rc == null) {
+ // Each sub-filter in filter list got true for filterAllRemaining().
+ return ReturnCode.SKIP;
+ } else {
+ return rc;
+ }
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell c) throws IOException {
+ if (isEmpty()) {
+ return ReturnCode.INCLUDE;
+ }
+ this.referenceCell = c;
+ return filterKeyValueWithMustPassOne(c);
+ }
+
+ @Override
+ public void reset() throws IOException {
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ filters.get(i).reset();
+ prevFilterRCList.set(i, null);
+ prevCellList.set(i, null);
+ }
+ }
+
+ @Override
+ public boolean filterRowKey(byte[] rowKey, int offset, int length) throws IOException {
+ if (isEmpty()) {
+ return super.filterRowKey(rowKey, offset, length);
+ }
+ boolean retVal = true;
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ Filter filter = filters.get(i);
+ if (!filter.filterAllRemaining() && !filter.filterRowKey(rowKey, offset, length)) {
+ retVal = false;
+ }
+ }
+ return retVal;
+ }
+
+ @Override
+ public boolean filterRowKey(Cell firstRowCell) throws IOException {
+ if (isEmpty()) {
+ return super.filterRowKey(firstRowCell);
+ }
+ boolean retVal = true;
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ Filter filter = filters.get(i);
+ if (!filter.filterAllRemaining() && !filter.filterRowKey(firstRowCell)) {
+ retVal = false;
+ }
+ }
+ return retVal;
+ }
+
+ @Override
+ public boolean filterAllRemaining() throws IOException {
+ if (isEmpty()) {
+ return super.filterAllRemaining();
+ }
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ if (!filters.get(i).filterAllRemaining()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public boolean filterRow() throws IOException {
+ if (isEmpty()) {
+ return super.filterRow();
+ }
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ Filter filter = filters.get(i);
+ if (!filter.filterRow()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public Cell getNextCellHint(Cell currentCell) throws IOException {
+ if (isEmpty()) {
+ return super.getNextCellHint(currentCell);
+ }
+ Cell minKeyHint = null;
+ // If any condition can pass, we need to keep the min hint
+ for (int i = 0, n = filters.size(); i < n; i++) {
+ if (filters.get(i).filterAllRemaining()) {
+ continue;
+ }
+ Cell curKeyHint = filters.get(i).getNextCellHint(currentCell);
+ if (curKeyHint == null) {
+ // If we ever don't have a hint and this is must-pass-one, then no hint
+ return null;
+ }
+ // If this is the first hint we find, set it
+ if (minKeyHint == null) {
+ minKeyHint = curKeyHint;
+ continue;
+ }
+ if (this.compareCell(minKeyHint, curKeyHint) > 0) {
+ minKeyHint = curKeyHint;
+ }
+ }
+ return minKeyHint;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/59275100/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index f7cba18..042ca08 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.testclassification.FilterTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Assert;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -799,5 +800,93 @@ public class TestFilterList {
filterList = new FilterList(Operator.MUST_PASS_ONE, filter6, filter6);
assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1));
}
+
+ static class MockSeekHintFilter extends FilterBase {
+ private Cell returnCell;
+
+ public MockSeekHintFilter(Cell returnCell) {
+ this.returnCell = returnCell;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell v) throws IOException {
+ return ReturnCode.SEEK_NEXT_USING_HINT;
+ }
+
+ @Override
+ public Cell getNextCellHint(Cell currentCell) throws IOException {
+ return this.returnCell;
+ }
+ }
+
+ @Test
+ public void testReversedFilterListWithMockSeekHintFilter() throws IOException {
+ KeyValue kv1 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
+ Bytes.toBytes("value"));
+ KeyValue kv2 = new KeyValue(Bytes.toBytes("row2"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
+ Bytes.toBytes("value"));
+ KeyValue kv3 = new KeyValue(Bytes.toBytes("row3"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
+ Bytes.toBytes("value"));
+ Filter filter1 = new MockSeekHintFilter(kv1);
+ filter1.setReversed(true);
+ Filter filter2 = new MockSeekHintFilter(kv2);
+ filter2.setReversed(true);
+ Filter filter3 = new MockSeekHintFilter(kv3);
+ filter3.setReversed(true);
+
+ FilterList filterList = new FilterList(Operator.MUST_PASS_ONE);
+ filterList.setReversed(true);
+ filterList.addFilter(filter1);
+ filterList.addFilter(filter2);
+ filterList.addFilter(filter3);
+
+ Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1));
+ Assert.assertEquals(kv3, filterList.getNextCellHint(kv1));
+
+ filterList = new FilterList(Operator.MUST_PASS_ALL);
+ filterList.setReversed(true);
+ filterList.addFilter(filter1);
+ filterList.addFilter(filter2);
+ filterList.addFilter(filter3);
+
+ Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterKeyValue(kv1));
+ Assert.assertEquals(kv1, filterList.getNextCellHint(kv1));
+ }
+
+ @Test
+ public void testReversedFilterListWithOR() throws IOException {
+ byte[] r22 = Bytes.toBytes("Row22");
+ byte[] r2 = Bytes.toBytes("Row2");
+ byte[] r1 = Bytes.toBytes("Row1");
+
+ FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
+ filterList.setReversed(true);
+ PrefixFilter prefixFilter = new PrefixFilter(r2);
+ prefixFilter.setReversed(true);
+ filterList.addFilter(prefixFilter);
+ filterList.filterRowKey(KeyValueUtil.createFirstOnRow(r22));
+ assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(new KeyValue(r22, r22, r22)));
+ assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(new KeyValue(r2, r2, r2)));
+
+ filterList.reset();
+ filterList.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
+ assertEquals(ReturnCode.SKIP, filterList.filterKeyValue(new KeyValue(r1, r1, r1)));
+
+ filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
+ filterList.setReversed(true);
+ AlwaysNextColFilter alwaysNextColFilter = new AlwaysNextColFilter();
+ alwaysNextColFilter.setReversed(true);
+ prefixFilter = new PrefixFilter(r2);
+ prefixFilter.setReversed(true);
+ filterList.addFilter(alwaysNextColFilter);
+ filterList.addFilter(prefixFilter);
+ filterList.filterRowKey(KeyValueUtil.createFirstOnRow(r22));
+ assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(new KeyValue(r22, r22, r22)));
+ assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(new KeyValue(r2, r2, r2)));
+
+ filterList.reset();
+ filterList.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
+ assertEquals(ReturnCode.NEXT_COL, filterList.filterKeyValue(new KeyValue(r1, r1, r1)));
+ }
}
[22/23] hbase git commit: HBASE-18904 Missing break in NEXT_ROW case
of FilterList#mergeReturnCodeForOrOperator()
Posted by zh...@apache.org.
HBASE-18904 Missing break in NEXT_ROW case of FilterList#mergeReturnCodeForOrOperator()
Signed-off-by: tedyu <yu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6442d30f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6442d30f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6442d30f
Branch: refs/heads/HBASE-18410
Commit: 6442d30f0f3b4bec791dfa2ebc0cad5938b3b18c
Parents: a811fb4
Author: Biju Nair <gs...@gmail.com>
Authored: Fri Sep 29 16:55:54 2017 -0400
Committer: zhangduo <zh...@apache.org>
Committed: Wed Oct 25 20:36:19 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hbase/filter/FilterList.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/6442d30f/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index 3147ab0..b518645 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -566,6 +566,7 @@ final public class FilterList extends FilterBase {
if (isInReturnCodes(rc, ReturnCode.NEXT_ROW)) {
return ReturnCode.NEXT_ROW;
}
+ break;
case SEEK_NEXT_USING_HINT:
if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
@@ -577,6 +578,7 @@ final public class FilterList extends FilterBase {
if (isInReturnCodes(rc, ReturnCode.SEEK_NEXT_USING_HINT)) {
return ReturnCode.SEEK_NEXT_USING_HINT;
}
+ break;
}
throw new IllegalStateException(
"Received code is not valid. rc: " + rc + ", localRC: " + localRC);
[06/23] hbase git commit: HBASE-19065 HRegion#bulkLoadHFiles() should
wait for concurrent Region#flush() to finish
Posted by zh...@apache.org.
HBASE-19065 HRegion#bulkLoadHFiles() should wait for concurrent Region#flush() to finish
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3cced094
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3cced094
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3cced094
Branch: refs/heads/HBASE-18410
Commit: 3cced094c57d7ecaf4f5b22a01cda58294595ced
Parents: cda2949
Author: tedyu <yu...@gmail.com>
Authored: Tue Oct 24 11:53:21 2017 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Tue Oct 24 11:53:21 2017 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hbase/regionserver/HRegion.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/3cced094/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index e03e4dd..5cbf889 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -5729,6 +5729,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
seqId = ((FlushResultImpl)fs).flushSequenceId;
} else if (fs.getResult() == FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY) {
seqId = ((FlushResultImpl)fs).flushSequenceId;
+ } else if (fs.getResult() == FlushResult.Result.CANNOT_FLUSH) {
+ // CANNOT_FLUSH may mean that a flush is already on-going
+ // we need to wait for that flush to complete
+ waitForFlushes();
} else {
throw new IOException("Could not bulk load with an assigned sequential ID because the "+
"flush didn't run. Reason for not flushing: " + ((FlushResultImpl)fs).failureReason);
[03/23] hbase git commit: HBASE-19021 Restore a few important missing
logics for balancer in 2.0
Posted by zh...@apache.org.
HBASE-19021 Restore a few important missing logics for balancer in 2.0
Signed-off-by: Jerry He <je...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9716f62f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9716f62f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9716f62f
Branch: refs/heads/HBASE-18410
Commit: 9716f62f43195ef024ac7a4bafb93a4716a7323e
Parents: 1c1906e
Author: Jerry He <je...@apache.org>
Authored: Tue Oct 24 07:53:17 2017 -0700
Committer: Jerry He <je...@apache.org>
Committed: Tue Oct 24 07:53:17 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/master/HMaster.java | 9 +++++---
.../hbase/master/assignment/RegionStates.java | 8 +++++++
.../master/procedure/ServerCrashProcedure.java | 1 +
.../hadoop/hbase/TestRegionRebalancing.java | 24 ++++++++++++++++----
4 files changed, 35 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9716f62f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 8f2ae6b..bb36520 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1421,16 +1421,19 @@ public class HMaster extends HRegionServer implements MasterServices {
}
}
+ boolean isByTable = getConfiguration().getBoolean("hbase.master.loadbalance.bytable", false);
Map<TableName, Map<ServerName, List<RegionInfo>>> assignmentsByTable =
- this.assignmentManager.getRegionStates().getAssignmentsByTable();
+ this.assignmentManager.getRegionStates().getAssignmentsByTable(!isByTable);
List<RegionPlan> plans = new ArrayList<>();
//Give the balancer the current cluster state.
this.balancer.setClusterStatus(getClusterStatus());
- this.balancer.setClusterLoad(
- this.assignmentManager.getRegionStates().getAssignmentsByTable());
+ this.balancer.setClusterLoad(assignmentsByTable);
+ for (Map<ServerName, List<RegionInfo>> serverMap : assignmentsByTable.values()) {
+ serverMap.keySet().removeAll(this.serverManager.getDrainingServersList());
+ }
for (Entry<TableName, Map<ServerName, List<RegionInfo>>> e : assignmentsByTable.entrySet()) {
List<RegionPlan> partialPlans = this.balancer.balanceCluster(e.getKey(), e.getValue());
if (partialPlans != null) plans.addAll(partialPlans);
http://git-wip-us.apache.org/repos/asf/hbase/blob/9716f62f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
index c13a49d..3b58fe2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
@@ -756,6 +756,14 @@ public class RegionStates {
serverResult.add(node.getRegionInfo());
}
+ // Add online servers with no assignment for the table.
+ for (Map<ServerName, List<RegionInfo>> table: result.values()) {
+ for (ServerName svr : serverMap.keySet()) {
+ if (!table.containsKey(svr)) {
+ table.put(svr, new ArrayList<RegionInfo>());
+ }
+ }
+ }
return result;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/9716f62f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
index a0ee628..56efaeb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
@@ -172,6 +172,7 @@ implements ServerProcedureInterface {
break;
case SERVER_CRASH_FINISH:
+ services.getAssignmentManager().getRegionStates().removeServer(serverName);
services.getServerManager().getDeadServers().finish(serverName);
return Flow.NO_MORE_STATE;
http://git-wip-us.apache.org/repos/asf/hbase/blob/9716f62f/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
index 3f7ea3b..cb9f768 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -53,9 +54,9 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
/**
* Test whether region re-balancing works. (HBASE-71)
+ * The test only works for cluster wide balancing, not per table wide.
+ * Increase the margin a little to make StochasticLoadBalancer result acceptable.
*/
-@Ignore // This is broken since new RegionServers does proper average of regions
-// and because Master is treated as a regionserver though it hosts two regions only.
@Category({FlakeyTests.class, LargeTests.class})
@RunWith(value = Parameterized.class)
public class TestRegionRebalancing {
@@ -131,6 +132,7 @@ public class TestRegionRebalancing {
// add a region server - total of 3
LOG.info("Started third server=" +
UTIL.getHBaseCluster().startRegionServer().getRegionServer().getServerName());
+ waitForAllRegionsAssigned();
assert(UTIL.getHBaseCluster().getMaster().balance() == true);
assertRegionsAreBalanced();
@@ -147,12 +149,14 @@ public class TestRegionRebalancing {
LOG.info("Added fourth server=" +
UTIL.getHBaseCluster().startRegionServer().getRegionServer().getServerName());
waitOnCrashProcessing();
+ waitForAllRegionsAssigned();
assert(UTIL.getHBaseCluster().getMaster().balance() == true);
assertRegionsAreBalanced();
for (int i = 0; i < 6; i++){
LOG.info("Adding " + (i + 5) + "th region server");
UTIL.getHBaseCluster().startRegionServer();
}
+ waitForAllRegionsAssigned();
assert(UTIL.getHBaseCluster().getMaster().balance() == true);
assertRegionsAreBalanced();
regionLocator.close();
@@ -188,9 +192,14 @@ public class TestRegionRebalancing {
long regionCount = UTIL.getMiniHBaseCluster().countServedRegions();
List<HRegionServer> servers = getOnlineRegionServers();
- double avg = UTIL.getHBaseCluster().getMaster().getAverageLoad();
+ double avg = (double)regionCount / (double)servers.size();
int avgLoadPlusSlop = (int)Math.ceil(avg * (1 + slop));
int avgLoadMinusSlop = (int)Math.floor(avg * (1 - slop)) - 1;
+ // Increase the margin a little to accommodate StochasticLoadBalancer
+ if (this.balancerName.contains("StochasticLoadBalancer")) {
+ avgLoadPlusSlop++;
+ avgLoadMinusSlop--;
+ }
LOG.debug("There are " + servers.size() + " servers and " + regionCount
+ " regions. Load Average: " + avg + " low border: " + avgLoadMinusSlop
+ ", up border: " + avgLoadPlusSlop + "; attempt: " + i);
@@ -250,13 +259,20 @@ public class TestRegionRebalancing {
*/
private void waitForAllRegionsAssigned() throws IOException {
int totalRegions = HBaseTestingUtility.KEYS.length;
+ try {
+ Thread.sleep(200);
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException();
+ }
while (UTIL.getMiniHBaseCluster().countServedRegions() < totalRegions) {
// while (!cluster.getMaster().allRegionsAssigned()) {
LOG.debug("Waiting for there to be "+ totalRegions +" regions, but there are "
+ UTIL.getMiniHBaseCluster().countServedRegions() + " right now.");
try {
Thread.sleep(200);
- } catch (InterruptedException e) {}
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException();
+ }
}
UTIL.waitUntilNoRegionsInTransition();
}
[16/23] hbase git commit: HBASE-17678 FilterList with MUST_PASS_ONE
lead to redundancy cells returned - addendum
Posted by zh...@apache.org.
HBASE-17678 FilterList with MUST_PASS_ONE lead to redundancy cells returned - addendum
Signed-off-by: tedyu <yu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5716e8d9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5716e8d9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5716e8d9
Branch: refs/heads/HBASE-18410
Commit: 5716e8d9ed5d3cc86ee6e03c57c1ce46cf18fa82
Parents: 4591098
Author: huzheng <op...@gmail.com>
Authored: Wed Jun 7 14:49:29 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Wed Oct 25 20:36:19 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/filter/FilterList.java | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/5716e8d9/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index 3493082..83db1f2 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.FilterProtos;
@@ -145,7 +146,7 @@ final public class FilterList extends FilterBase {
public void initPrevListForMustPassOne(int size) {
if (operator == Operator.MUST_PASS_ONE) {
- if (this.prevCellList == null) {
+ if (this.prevFilterRCList == null) {
prevFilterRCList = new ArrayList<>(Collections.nCopies(size, null));
}
if (this.prevCellList == null) {
@@ -407,7 +408,14 @@ final public class FilterList extends FilterBase {
ReturnCode localRC = filter.filterKeyValue(c);
// Update previous cell and return code we encountered.
prevFilterRCList.set(i, localRC);
- prevCellList.set(i, c);
+ if (c == null || localRC == ReturnCode.INCLUDE || localRC == ReturnCode.SKIP) {
+ // If previous return code is INCLUDE or SKIP, we should always pass the next cell to the
+ // corresponding sub-filter(need not test shouldPassCurrentCellToFilter() method), So we
+ // need not save current cell to prevCellList for saving heap memory.
+ prevCellList.set(i, null);
+ } else {
+ prevCellList.set(i, KeyValueUtil.toNewKeyCell(c));
+ }
if (localRC != ReturnCode.SEEK_NEXT_USING_HINT) {
seenNonHintReturnCode = true;
[21/23] hbase git commit: HBASE-18368 FilterList with multiple
FamilyFilters concatenated by OR does not work
Posted by zh...@apache.org.
HBASE-18368 FilterList with multiple FamilyFilters concatenated by OR does not work
Signed-off-by: Guanghao Zhang <zg...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fcaf71d2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fcaf71d2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fcaf71d2
Branch: refs/heads/HBASE-18410
Commit: fcaf71d20687e350d9357e1a303da6caf4c6ac08
Parents: 8a5114b
Author: huzheng <op...@gmail.com>
Authored: Tue Oct 17 19:25:23 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Wed Oct 25 20:36:19 2017 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/filter/Filter.java | 10 +++++---
.../hadoop/hbase/filter/FilterListWithOR.java | 10 ++++++--
.../hadoop/hbase/filter/TestFilterList.java | 26 ++++++++++++++++++++
.../hbase/filter/TestFilterListOnMini.java | 7 +++---
4 files changed, 44 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/fcaf71d2/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/Filter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
index 70c68b6..a92ea0b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
@@ -172,8 +172,12 @@ public abstract class Filter {
*/
NEXT_COL,
/**
- * Done with columns, skip to next row. Note that filterRow() will
- * still be called.
+ * Seek to next row in current family. It may still pass a cell whose family is different but
+ * row is the same as previous cell to {@link #filterKeyValue(Cell)} , even if we get a NEXT_ROW
+ * returned for previous cell. For more details see HBASE-18368. <br>
+ * Once reset() method was invoked, then we switch to the next row for all family, and you can
+ * catch the event by invoking CellUtils.matchingRows(previousCell, currentCell). <br>
+ * Note that filterRow() will still be called. <br>
*/
NEXT_ROW,
/**
@@ -181,7 +185,7 @@ public abstract class Filter {
*/
SEEK_NEXT_USING_HINT,
/**
- * Include KeyValue and done with row, seek to next.
+ * Include KeyValue and done with row, seek to next. See NEXT_ROW.
*/
INCLUDE_AND_SEEK_NEXT_ROW,
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fcaf71d2/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java
index bac9023..31e2a55 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java
@@ -74,7 +74,12 @@ public class FilterListWithOR extends FilterListBase {
* as the previous cell even if filter-A has NEXT_COL returned for the previous cell. So we should
* save the previous cell and the return code list when checking previous cell for every filter in
* filter list, and verify if currentCell fit the previous return code, if fit then pass the
- * currentCell to the corresponding filter. (HBASE-17678)
+ * currentCell to the corresponding filter. (HBASE-17678) <br>
+ * Note that: In StoreScanner level, NEXT_ROW will skip to the next row in current family, and in
+ * RegionScanner level, NEXT_ROW will skip to the next row in current family and switch to the
+ * next family for RegionScanner, INCLUDE_AND_NEXT_ROW is the same. so we should pass current cell
+ * to the filter, if row mismatch or row match but column family mismatch. (HBASE-18368)
+ * @see org.apache.hadoop.hbase.filter.Filter.ReturnCode
*/
private boolean shouldPassCurrentCellToFilter(Cell prevCell, Cell currentCell, int filterIdx)
throws IOException {
@@ -94,7 +99,8 @@ public class FilterListWithOR extends FilterListBase {
return !CellUtil.matchingRowColumn(prevCell, currentCell);
case NEXT_ROW:
case INCLUDE_AND_SEEK_NEXT_ROW:
- return !CellUtil.matchingRows(prevCell, currentCell);
+ return !CellUtil.matchingRows(prevCell, currentCell)
+ || !CellUtil.matchingFamily(prevCell, currentCell);
default:
throw new IllegalStateException("Received code is not valid.");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fcaf71d2/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index 4c67c59..d420b02 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -936,5 +936,31 @@ public class TestFilterList {
assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(kv));
assertEquals(kv, filterList.transformCell(kv));
}
+
+ private static class MockNextRowFilter extends FilterBase {
+ private int hitCount = 0;
+
+ public ReturnCode filterKeyValue(Cell v) throws IOException {
+ hitCount++;
+ return ReturnCode.NEXT_ROW;
+ }
+
+ public int getHitCount() {
+ return hitCount;
+ }
+ }
+
+ @Test
+ public void testRowCountFilter() throws IOException {
+ KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam1"), Bytes.toBytes("a"), 1,
+ Bytes.toBytes("value"));
+ KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam2"), Bytes.toBytes("a"), 2,
+ Bytes.toBytes("value"));
+ MockNextRowFilter mockNextRowFilter = new MockNextRowFilter();
+ FilterList filter = new FilterList(Operator.MUST_PASS_ONE, mockNextRowFilter);
+ filter.filterKeyValue(kv1);
+ filter.filterKeyValue(kv2);
+ assertEquals(2, mockNextRowFilter.getHitCount());
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fcaf71d2/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterListOnMini.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterListOnMini.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterListOnMini.java
index 590b26e..2a13ac8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterListOnMini.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterListOnMini.java
@@ -36,10 +36,10 @@ import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
/**
- * Tests filter Lists in ways that rely on a MiniCluster.
- * Where possible, favor tests in TestFilterList and TestFilterFromRegionSide instead.
+ * Tests filter Lists in ways that rely on a MiniCluster. Where possible, favor tests in
+ * TestFilterList and TestFilterFromRegionSide instead.
*/
-@Category({MediumTests.class, FilterTests.class})
+@Category({ MediumTests.class, FilterTests.class })
public class TestFilterListOnMini {
private static final Log LOG = LogFactory.getLog(TestFilterListOnMini.class);
@@ -58,7 +58,6 @@ public class TestFilterListOnMini {
TEST_UTIL.shutdownMiniCluster();
}
- @Ignore("HBASE-18410 Should not merge without this test running.")
@Test
public void testFiltersWithOR() throws Exception {
TableName tn = TableName.valueOf(name.getMethodName());
[09/23] hbase git commit: HBASE-19073 Cleanup CoordinatedStateManager
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
index d23dd1a..e420959 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
@@ -906,7 +906,7 @@ public abstract class AbstractTestWALReplay {
assertNotNull(listStatus);
assertTrue(listStatus.length > 0);
WALSplitter.splitLogFile(hbaseRootDir, listStatus[0],
- this.fs, this.conf, null, null, null, mode, wals);
+ this.fs, this.conf, null, null, null, null, mode, wals);
FileStatus[] listStatus1 = this.fs.listStatus(
new Path(FSUtils.getTableDir(hbaseRootDir, tableName), new Path(hri.getEncodedName(),
"recovered.edits")), new PathFilter() {
@@ -1059,9 +1059,9 @@ public abstract class AbstractTestWALReplay {
first = fs.getFileStatus(smallFile);
second = fs.getFileStatus(largeFile);
}
- WALSplitter.splitLogFile(hbaseRootDir, first, fs, conf, null, null, null,
+ WALSplitter.splitLogFile(hbaseRootDir, first, fs, conf, null, null, null, null,
RecoveryMode.LOG_SPLITTING, wals);
- WALSplitter.splitLogFile(hbaseRootDir, second, fs, conf, null, null, null,
+ WALSplitter.splitLogFile(hbaseRootDir, second, fs, conf, null, null, null, null,
RecoveryMode.LOG_SPLITTING, wals);
WAL wal = createWAL(this.conf, hbaseRootDir, logName);
region = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal);
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
index ebb1bf8..71b4def 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
@@ -277,11 +277,6 @@ public class TestReplicationSource {
super(conf);
}
- public ShutdownDelayRegionServer(Configuration conf, CoordinatedStateManager csm)
- throws IOException, InterruptedException {
- super(conf, csm);
- }
-
@Override
protected void stopServiceThreads() {
// Add a delay before service threads are shutdown.
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java
index 570353b..c7203b3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java
@@ -134,7 +134,7 @@ public class TestWALReaderOnSecureWAL {
wal.sync();
final Path walPath = AbstractFSWALProvider.getCurrentFileName(wal);
wal.shutdown();
-
+
return walPath;
} finally {
// restore the cell codec class
@@ -182,11 +182,11 @@ public class TestWALReaderOnSecureWAL {
}
FileStatus[] listStatus = fs.listStatus(walPath.getParent());
- RecoveryMode mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ?
+ RecoveryMode mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ?
RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
Path rootdir = FSUtils.getRootDir(conf);
try {
- WALSplitter s = new WALSplitter(wals, conf, rootdir, fs, null, null, mode);
+ WALSplitter s = new WALSplitter(wals, conf, rootdir, fs, null, null, null, mode);
s.splitLogFile(listStatus[0], null);
Path file = new Path(ZKSplitLog.getSplitLogDir(rootdir, listStatus[0].getPath().getName()),
"corrupt");
@@ -229,11 +229,11 @@ public class TestWALReaderOnSecureWAL {
}
FileStatus[] listStatus = fs.listStatus(walPath.getParent());
- RecoveryMode mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ?
+ RecoveryMode mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ?
RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
Path rootdir = FSUtils.getRootDir(conf);
try {
- WALSplitter s = new WALSplitter(wals, conf, rootdir, fs, null, null, mode);
+ WALSplitter s = new WALSplitter(wals, conf, rootdir, fs, null, null, null, mode);
s.splitLogFile(listStatus[0], null);
Path file = new Path(ZKSplitLog.getSplitLogDir(rootdir, listStatus[0].getPath().getName()),
"corrupt");
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
index b988544..b3fd308 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
@@ -372,8 +372,7 @@ public class TestWALSplit {
}
/**
- * @throws IOException
- * @see https://issues.apache.org/jira/browse/HBASE-3020
+ * {@see https://issues.apache.org/jira/browse/HBASE-3020}
*/
@Test (timeout=300000)
public void testRecoveredEditsPathForMeta() throws IOException {
@@ -805,7 +804,7 @@ public class TestWALSplit {
assertTrue("There should be some log greater than size 0.", 0 < largestSize);
// Set up a splitter that will throw an IOE on the output side
WALSplitter logSplitter = new WALSplitter(wals,
- conf, HBASEDIR, fs, null, null, this.mode) {
+ conf, HBASEDIR, fs, null, null, null, this.mode) {
@Override
protected Writer createWriter(Path logfile) throws IOException {
Writer mockWriter = Mockito.mock(Writer.class);
@@ -932,7 +931,7 @@ public class TestWALSplit {
try {
conf.setInt("hbase.splitlog.report.period", 1000);
boolean ret = WALSplitter.splitLogFile(
- HBASEDIR, logfile, spiedFs, conf, localReporter, null, null, this.mode, wals);
+ HBASEDIR, logfile, spiedFs, conf, localReporter, null, null, null, this.mode, wals);
assertFalse("Log splitting should failed", ret);
assertTrue(count.get() > 0);
} catch (IOException e) {
@@ -991,7 +990,7 @@ public class TestWALSplit {
// Create a splitter that reads and writes the data without touching disk
WALSplitter logSplitter = new WALSplitter(wals,
- localConf, HBASEDIR, fs, null, null, this.mode) {
+ localConf, HBASEDIR, fs, null, null, null, this.mode) {
/* Produce a mock writer that doesn't write anywhere */
@Override
@@ -1123,8 +1122,7 @@ public class TestWALSplit {
}
/**
- * @throws IOException
- * @see https://issues.apache.org/jira/browse/HBASE-4862
+ * {@see https://issues.apache.org/jira/browse/HBASE-4862}
*/
@Test (timeout=300000)
public void testConcurrentSplitLogAndReplayRecoverEdit() throws IOException {
@@ -1142,7 +1140,7 @@ public class TestWALSplit {
logfiles != null && logfiles.length > 0);
WALSplitter logSplitter = new WALSplitter(wals,
- conf, HBASEDIR, fs, null, null, this.mode) {
+ conf, HBASEDIR, fs, null, null, null, this.mode) {
@Override
protected Writer createWriter(Path logfile)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd70cc30/src/main/asciidoc/_chapters/hbase-default.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/hbase-default.adoc b/src/main/asciidoc/_chapters/hbase-default.adoc
index 32d114e..2093c57 100644
--- a/src/main/asciidoc/_chapters/hbase-default.adoc
+++ b/src/main/asciidoc/_chapters/hbase-default.adoc
@@ -2024,17 +2024,6 @@ A comma-separated list of
.Default
``
-
-[[hbase.coordinated.state.manager.class]]
-*`hbase.coordinated.state.manager.class`*::
-+
-.Description
-Fully qualified name of class implementing coordinated state manager.
-+
-.Default
-`org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager`
-
-
[[hbase.regionserver.storefile.refresh.period]]
*`hbase.regionserver.storefile.refresh.period`*::
+
[15/23] hbase git commit: HBASE-18879 HBase FilterList cause
KeyOnlyFilter not work
Posted by zh...@apache.org.
HBASE-18879 HBase FilterList cause KeyOnlyFilter not work
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8a5114b4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8a5114b4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8a5114b4
Branch: refs/heads/HBASE-18410
Commit: 8a5114b4fa079ff165ba735691eb4d99e3b5d7ca
Parents: 5927510
Author: huzheng <op...@gmail.com>
Authored: Wed Oct 11 21:17:03 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Wed Oct 25 20:36:19 2017 +0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/filter/FilterList.java | 6 +++
.../hadoop/hbase/filter/FilterListBase.java | 3 ++
.../hadoop/hbase/filter/FilterListWithAND.java | 22 +++++----
.../hadoop/hbase/filter/FilterListWithOR.java | 22 +++++----
.../hadoop/hbase/filter/TestFilterList.java | 48 ++++++++++++++++++++
5 files changed, 85 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8a5114b4/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index 97392d1..e87f1b3 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -72,6 +72,8 @@ final public class FilterList extends FilterBase {
filterListBase = new FilterListWithAND(filters);
} else if (operator == Operator.MUST_PASS_ONE) {
filterListBase = new FilterListWithOR(filters);
+ } else {
+ throw new IllegalArgumentException("Invalid operator: " + operator);
}
this.operator = operator;
}
@@ -168,6 +170,10 @@ final public class FilterList extends FilterBase {
return filterListBase.transformCell(c);
}
+ ReturnCode internalFilterKeyValue(Cell c, Cell currentTransformedCell) throws IOException {
+ return this.filterListBase.internalFilterKeyValue(c, currentTransformedCell);
+ }
+
@Override
public ReturnCode filterKeyValue(Cell c) throws IOException {
return filterListBase.filterKeyValue(c);
http://git-wip-us.apache.org/repos/asf/hbase/blob/8a5114b4/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java
index 7fa0245..60b0dc1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListBase.java
@@ -107,6 +107,9 @@ public abstract class FilterListBase extends FilterBase {
return cell;
}
+ abstract ReturnCode internalFilterKeyValue(Cell c, Cell currentTransformedCell)
+ throws IOException;
+
/**
* Filters that never filter by modifying the returned List of Cells can inherit this
* implementation that does nothing. {@inheritDoc}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8a5114b4/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java
index fa979c0..4909dfd 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java
@@ -147,16 +147,26 @@ public class FilterListWithAND extends FilterListBase {
"Received code is not valid. rc: " + rc + ", localRC: " + localRC);
}
- private ReturnCode filterKeyValueWithMustPassAll(Cell c) throws IOException {
+ @Override
+ ReturnCode internalFilterKeyValue(Cell c, Cell currentTransformedCell) throws IOException {
+ if (isEmpty()) {
+ return ReturnCode.INCLUDE;
+ }
ReturnCode rc = ReturnCode.INCLUDE;
- Cell transformed = c;
+ Cell transformed = currentTransformedCell;
+ this.referenceCell = c;
this.seekHintFilter.clear();
for (int i = 0, n = filters.size(); i < n; i++) {
Filter filter = filters.get(i);
if (filter.filterAllRemaining()) {
return ReturnCode.NEXT_ROW;
}
- ReturnCode localRC = filter.filterKeyValue(c);
+ ReturnCode localRC;
+ if (filter instanceof FilterList) {
+ localRC = ((FilterList) filter).internalFilterKeyValue(c, transformed);
+ } else {
+ localRC = filter.filterKeyValue(c);
+ }
rc = mergeReturnCode(rc, localRC);
// For INCLUDE* case, we need to update the transformed cell.
@@ -177,11 +187,7 @@ public class FilterListWithAND extends FilterListBase {
@Override
public ReturnCode filterKeyValue(Cell c) throws IOException {
- if (isEmpty()) {
- return ReturnCode.INCLUDE;
- }
- this.referenceCell = c;
- return filterKeyValueWithMustPassAll(c);
+ return internalFilterKeyValue(c, c);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/8a5114b4/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java
index 55306f2..bac9023 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithOR.java
@@ -235,10 +235,15 @@ public class FilterListWithOR extends FilterListBase {
}
}
- private ReturnCode filterKeyValueWithMustPassOne(Cell c) throws IOException {
+ @Override
+ ReturnCode internalFilterKeyValue(Cell c, Cell currentTransformCell) throws IOException {
+ if (isEmpty()) {
+ return ReturnCode.INCLUDE;
+ }
ReturnCode rc = null;
boolean everyFilterReturnHint = true;
- Cell transformed = c;
+ Cell transformed = currentTransformCell;
+ this.referenceCell = c;
for (int i = 0, n = filters.size(); i < n; i++) {
Filter filter = filters.get(i);
@@ -248,7 +253,12 @@ public class FilterListWithOR extends FilterListBase {
continue;
}
- ReturnCode localRC = filter.filterKeyValue(c);
+ ReturnCode localRC;
+ if (filter instanceof FilterList) {
+ localRC = ((FilterList) filter).internalFilterKeyValue(c, transformed);
+ } else {
+ localRC = filter.filterKeyValue(c);
+ }
// Update previous return code and previous cell for filter[i].
updatePrevFilterRCList(i, localRC);
@@ -280,11 +290,7 @@ public class FilterListWithOR extends FilterListBase {
@Override
public ReturnCode filterKeyValue(Cell c) throws IOException {
- if (isEmpty()) {
- return ReturnCode.INCLUDE;
- }
- this.referenceCell = c;
- return filterKeyValueWithMustPassOne(c);
+ return internalFilterKeyValue(c, c);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/8a5114b4/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index 042ca08..4c67c59 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -888,5 +888,53 @@ public class TestFilterList {
filterList.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
assertEquals(ReturnCode.NEXT_COL, filterList.filterKeyValue(new KeyValue(r1, r1, r1)));
}
+
+ @Test
+ public void testKeyOnlyFilterTransformCell() throws IOException {
+ Cell c;
+ KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
+ 1, Bytes.toBytes("value1"));
+ KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
+ 2, Bytes.toBytes("value2"));
+
+ Filter filter1 = new SingleColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("column1"),
+ CompareOperator.EQUAL, Bytes.toBytes("value1"));
+ Filter filter2 = new SingleColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("column1"),
+ CompareOperator.EQUAL, Bytes.toBytes("value2"));
+ FilterList internalFilterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2);
+
+ FilterList keyOnlyFilterFirst =
+ new FilterList(Operator.MUST_PASS_ALL, new KeyOnlyFilter(), internalFilterList);
+
+ assertEquals(ReturnCode.INCLUDE, keyOnlyFilterFirst.filterKeyValue(kv1));
+ c = keyOnlyFilterFirst.transformCell(kv1);
+ assertEquals(0, c.getValueLength());
+ assertEquals(ReturnCode.INCLUDE, keyOnlyFilterFirst.filterKeyValue(kv2));
+ c = keyOnlyFilterFirst.transformCell(kv2);
+ assertEquals(0, c.getValueLength());
+
+ internalFilterList.reset();
+ FilterList keyOnlyFilterLast =
+ new FilterList(Operator.MUST_PASS_ALL, new KeyOnlyFilter(), internalFilterList);
+ assertEquals(ReturnCode.INCLUDE, keyOnlyFilterLast.filterKeyValue(kv1));
+ c = keyOnlyFilterLast.transformCell(kv1);
+ assertEquals(0, c.getValueLength());
+ assertEquals(ReturnCode.INCLUDE, keyOnlyFilterLast.filterKeyValue(kv2));
+ c = keyOnlyFilterLast.transformCell(kv2);
+ assertEquals(0, c.getValueLength());
+ }
+
+ @Test
+ public void testEmptyFilterListTransformCell() throws IOException {
+ KeyValue kv = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
+ 1, Bytes.toBytes("value"));
+ FilterList filterList = new FilterList(Operator.MUST_PASS_ALL);
+ assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(kv));
+ assertEquals(kv, filterList.transformCell(kv));
+
+ filterList = new FilterList(Operator.MUST_PASS_ONE);
+ assertEquals(ReturnCode.INCLUDE, filterList.filterKeyValue(kv));
+ assertEquals(kv, filterList.transformCell(kv));
+ }
}
[02/23] hbase git commit: HBASE-19049 Update Kerby to 1.0.1.
Posted by zh...@apache.org.
HBASE-19049 Update Kerby to 1.0.1.
Signed-off-by: Mike Drob <md...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1c1906e0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1c1906e0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1c1906e0
Branch: refs/heads/HBASE-18410
Commit: 1c1906e03acefab82df4d7c08e1684fe08409406
Parents: a31e8e3
Author: Sean Busbey <bu...@apache.org>
Authored: Thu Oct 19 09:05:50 2017 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Tue Oct 24 09:43:20 2017 -0500
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/1c1906e0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 60599c6..6e3bfb3 100755
--- a/pom.xml
+++ b/pom.xml
@@ -1440,7 +1440,7 @@
<jcodings.version>1.0.18</jcodings.version>
<spy.version>2.12.2</spy.version>
<bouncycastle.version>1.46</bouncycastle.version>
- <kerby.version>1.0.0-RC2</kerby.version>
+ <kerby.version>1.0.1</kerby.version>
<commons-crypto.version>1.0.0</commons-crypto.version>
<curator.version>4.0.0</curator.version>
<!-- Plugin Dependencies -->
[14/23] hbase git commit: HBASE-15410 Utilize the max seek value when
all Filters in MUST_PASS_ALL FilterList return SEEK_NEXT_USING_HINT
Posted by zh...@apache.org.
HBASE-15410 Utilize the max seek value when all Filters in MUST_PASS_ALL FilterList return SEEK_NEXT_USING_HINT
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/10612254
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/10612254
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/10612254
Branch: refs/heads/HBASE-18410
Commit: 10612254a163d39e0ad70af683fe1890125c41ed
Parents: 5716e8d
Author: tedyu <yu...@gmail.com>
Authored: Thu Sep 7 04:07:09 2017 -0700
Committer: zhangduo <zh...@apache.org>
Committed: Wed Oct 25 20:36:19 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hbase/filter/FilterList.java | 5 +++--
.../java/org/apache/hadoop/hbase/filter/TestFilterList.java | 4 ++--
2 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/10612254/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
index 83db1f2..3ff978d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
@@ -28,12 +28,13 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.FilterProtos;
-import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
/**
* Implementation of {@link Filter} that represents an ordered List of Filters
http://git-wip-us.apache.org/repos/asf/hbase/blob/10612254/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index 46d44de..e414729 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -502,8 +502,8 @@ public class TestFilterList {
// Should take the min if given two hints
FilterList filterList = new FilterList(Operator.MUST_PASS_ONE,
Arrays.asList(new Filter [] { filterMinHint, filterMaxHint } ));
- assertEquals(0,
- CellComparatorImpl.COMPARATOR.compare(filterList.getNextCellHint(null), minKeyValue));
+ assertEquals(0, CellComparatorImpl.COMPARATOR.compare(filterList.getNextCellHint(null),
+ minKeyValue));
// Should have no hint if any filter has no hint
filterList = new FilterList(Operator.MUST_PASS_ONE,