You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2017/10/24 03:45:11 UTC
[31/50] [abbrv] hbase git commit: HBASE-19007 Align Services
Interfaces in Master and RegionServer
HBASE-19007 Align Services Interfaces in Master and RegionServer
Purges Server, MasterServices, and RegionServerServices from
CoprocessorEnvironments. Replaces removed functionality with
a set of carefully curated methods on the *CoprocessorEnvironment
implementations (Varies by CoprocessorEnvironment in that the
MasterCoprocessorEnvironment has Master-type facility exposed,
and so on).
A few core Coprocessors that should long ago have been converted
to be integral, violate their context; e.g. a RegionCoprocessor
wants free access to a hosting RegionServer (which may or may not
be present). Rather than let these violators make us corrupte the
CP API, instead, we've made up a hacky system that allows core
Coprocessors access to internals. A new CoreCoprocessor Annotation
has been introduced. When loading Coprocessors, if the instance is
annotated CoreCoprocessor, we pass it an Environment that has been
padded w/ extra-stuff. On invocation, CoreCoprocessors know how to
route their way to these extras in their environment.
See the *CoprocessoHost for how the do the check for CoreCoprocessor
and pass a fatter *Coprocessor, one that allows getting of either
a RegionServerService or MasterService out of the environment
via Marker Interfaces.
Removed org.apache.hadoop.hbase.regionserver.CoprocessorRegionServerServices
M hbase-endpoint/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
This Endpoint has been deprecated because its functionality has been
moved to core. Marking it a CoreCoprocessor in the meantime to
minimize change.
M hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
This should be integral to hbase. Meantime, marking it CoreCoprocessor.
M hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java
Added doc on where it is used and added back a few methods we'd
removed.
A hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoreCoprocessor.java
New annotation for core hbase coprocessors. They get richer environment
on coprocessor loading.
A hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/HasMasterServices.java
A hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/HasRegionServerServices.java
Marker Interface to access extras if present.
M hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java
Purge MasterServices access. Allow CPs a Connection.
M hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java
Purge RegionServerServices access. Allow CPs a Connection.
M hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java
Purge MasterServices access. Allow CPs a Connection.
M hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
M hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
We no longer have access to MasterServices. Don't need it actually.
Use short-circuiting Admin instead.
D hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CoprocessorRegionServerServices.java
Removed. Not needed now we do CP Env differently.
M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
No need to go via RSS to getOnlineTables; just use HRS.
And so on. Adds tests to ensure we can only get at extra info
if the CP has been properly marked.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/38879fb3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/38879fb3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/38879fb3
Branch: refs/heads/HBASE-18410
Commit: 38879fb3ffa88ca95b15c61656a92e72c0ed996f
Parents: 592d541
Author: Guanghao Zhang <zg...@apache.org>
Authored: Mon Oct 16 17:12:37 2017 +0800
Committer: Michael Stack <st...@apache.org>
Committed: Sat Oct 21 11:06:30 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hbase/client/Connection.java | 2 +-
.../hadoop/hbase/client/ConnectionUtils.java | 64 +++++++----
.../apache/hadoop/hbase/client/HBaseAdmin.java | 2 +-
.../security/access/SecureBulkLoadEndpoint.java | 6 +-
.../hbase/rsgroup/RSGroupAdminEndpoint.java | 10 +-
.../java/org/apache/hadoop/hbase/Server.java | 21 +++-
.../hbase/coprocessor/BaseEnvironment.java | 3 +-
.../hbase/coprocessor/CoreCoprocessor.java | 45 ++++++++
.../hbase/coprocessor/HasMasterServices.java | 37 ++++++
.../coprocessor/HasRegionServerServices.java | 37 ++++++
.../MasterCoprocessorEnvironment.java | 24 +++-
.../RegionCoprocessorEnvironment.java | 26 ++++-
.../RegionServerCoprocessorEnvironment.java | 24 +++-
.../org/apache/hadoop/hbase/ipc/RpcServer.java | 9 +-
.../hbase/master/MasterCoprocessorHost.java | 48 ++++++--
.../hadoop/hbase/master/MasterServices.java | 12 +-
.../hbase/quotas/MasterSpaceQuotaObserver.java | 11 +-
.../apache/hadoop/hbase/quotas/QuotaCache.java | 5 +-
.../CoprocessorRegionServerServices.java | 64 -----------
.../hadoop/hbase/regionserver/HRegion.java | 2 +-
.../hbase/regionserver/HRegionServer.java | 4 +-
.../regionserver/RegionCoprocessorHost.java | 57 ++++++++--
.../RegionServerCoprocessorHost.java | 55 +++++++--
.../regionserver/RegionServerServices.java | 8 +-
.../regionserver/ReplicationObserver.java | 24 ++--
.../regionserver/ReplicationSyncUp.java | 12 +-
.../security/access/AccessControlLists.java | 28 +----
.../hbase/security/access/AccessController.java | 99 ++++++++++------
.../CoprocessorWhitelistMasterObserver.java | 4 +-
.../hbase/security/token/TokenProvider.java | 15 ++-
.../DefaultVisibilityLabelServiceImpl.java | 17 ++-
.../visibility/VisibilityController.java | 13 ++-
.../VisibilityLabelServiceManager.java | 2 +-
.../hadoop/hbase/MockRegionServerServices.java | 7 +-
.../hbase/client/TestReplicaWithCluster.java | 21 ++--
.../coprocessor/TestCoprocessorInterface.java | 10 +-
.../TestCoprocessorShortCircuitRPC.java | 114 +++++++++++++++++++
.../coprocessor/TestCoreMasterCoprocessor.java | 99 ++++++++++++++++
.../coprocessor/TestCoreRegionCoprocessor.java | 113 ++++++++++++++++++
.../TestCoreRegionServerCoprocessor.java | 99 ++++++++++++++++
.../coprocessor/TestOpenTableInCoprocessor.java | 7 +-
.../coprocessor/TestRegionObserverStacking.java | 8 +-
...gionServerCoprocessorExceptionWithAbort.java | 2 +-
.../hbase/master/MockNoopMasterServices.java | 8 +-
.../hadoop/hbase/master/MockRegionServer.java | 8 +-
.../hbase/master/TestActiveMasterManager.java | 13 ++-
.../hbase/master/cleaner/TestHFileCleaner.java | 13 ++-
.../master/cleaner/TestHFileLinkCleaner.java | 12 +-
.../hbase/master/cleaner/TestLogsCleaner.java | 12 +-
.../cleaner/TestReplicationHFileCleaner.java | 12 +-
.../regionserver/TestHeapMemoryManager.java | 13 ++-
.../regionserver/TestRegionServerAbort.java | 14 ++-
.../hbase/regionserver/TestSplitLogWorker.java | 11 ++
.../hbase/regionserver/TestWALLockup.java | 11 +-
.../TestReplicationStateHBaseImpl.java | 12 +-
.../replication/TestReplicationStateZKImpl.java | 11 ++
.../TestReplicationTrackerZKImpl.java | 13 ++-
.../TestReplicationSourceManager.java | 11 +-
.../security/token/TestTokenAuthentication.java | 26 ++++-
.../apache/hadoop/hbase/util/MockServer.java | 13 ++-
.../hbase/thrift/ErrorThrowingGetObserver.java | 5 +-
61 files changed, 1180 insertions(+), 308 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Connection.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Connection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Connection.java
index 66e79be..1d28777 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Connection.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Connection.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java
index 9c7fd34..4f22402 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -124,6 +124,46 @@ public final class ConnectionUtils {
}
/**
+ * A ClusterConnection that will short-circuit RPC making direct invocations against the
+ * localhost if the invocation target is 'this' server; save on network and protobuf
+ * invocations.
+ */
+ @VisibleForTesting // Class is visible so can assert we are short-circuiting when expected.
+ public static class ShortCircuitingClusterConnection extends ConnectionImplementation {
+ private final ServerName serverName;
+ private final AdminService.BlockingInterface localHostAdmin;
+ private final ClientService.BlockingInterface localHostClient;
+
+ private ShortCircuitingClusterConnection(Configuration conf, ExecutorService pool, User user,
+ ServerName serverName, AdminService.BlockingInterface admin,
+ ClientService.BlockingInterface client)
+ throws IOException {
+ super(conf, pool, user);
+ this.serverName = serverName;
+ this.localHostAdmin = admin;
+ this.localHostClient = client;
+ }
+
+ @Override
+ public AdminService.BlockingInterface getAdmin(ServerName sn) throws IOException {
+ return serverName.equals(sn) ? this.localHostAdmin : super.getAdmin(sn);
+ }
+
+ @Override
+ public ClientService.BlockingInterface getClient(ServerName sn) throws IOException {
+ return serverName.equals(sn) ? this.localHostClient : super.getClient(sn);
+ }
+
+ @Override
+ public MasterKeepAliveConnection getKeepAliveMasterService() throws MasterNotRunningException {
+ if (this.localHostClient instanceof MasterService.BlockingInterface) {
+ return new ShortCircuitMasterConnection((MasterService.BlockingInterface)this.localHostClient);
+ }
+ return super.getKeepAliveMasterService();
+ }
+ }
+
+ /**
* Creates a short-circuit connection that can bypass the RPC layer (serialization,
* deserialization, networking, etc..) when talking to a local server.
* @param conf the current configuration
@@ -142,27 +182,7 @@ public final class ConnectionUtils {
if (user == null) {
user = UserProvider.instantiate(conf).getCurrent();
}
- return new ConnectionImplementation(conf, pool, user) {
- @Override
- public AdminService.BlockingInterface getAdmin(ServerName sn) throws IOException {
- return serverName.equals(sn) ? admin : super.getAdmin(sn);
- }
-
- @Override
- public ClientService.BlockingInterface getClient(ServerName sn) throws IOException {
- return serverName.equals(sn) ? client : super.getClient(sn);
- }
-
- @Override
- public MasterKeepAliveConnection getKeepAliveMasterService()
- throws MasterNotRunningException {
- if (!(client instanceof MasterService.BlockingInterface)) {
- return super.getKeepAliveMasterService();
- } else {
- return new ShortCircuitMasterConnection((MasterService.BlockingInterface) client);
- }
- }
- };
+ return new ShortCircuitingClusterConnection(conf, pool, user, serverName, admin, client);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 5ca6f2f..e4bb675 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-endpoint/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-endpoint/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java b/hbase-endpoint/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
index f9798aa..69d8491 100644
--- a/hbase-endpoint/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
+++ b/hbase-endpoint/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java
@@ -27,6 +27,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.HasRegionServerServices;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
@@ -54,6 +56,7 @@ import org.apache.yetus.audience.InterfaceAudience;
* Coprocessor service for bulk loads in secure mode.
* @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
*/
+@CoreCoprocessor
@InterfaceAudience.Private
@Deprecated
public class SecureBulkLoadEndpoint extends SecureBulkLoadService implements RegionCoprocessor {
@@ -68,8 +71,7 @@ public class SecureBulkLoadEndpoint extends SecureBulkLoadService implements Reg
@Override
public void start(CoprocessorEnvironment env) {
this.env = (RegionCoprocessorEnvironment)env;
- assert this.env.getCoprocessorRegionServerServices() instanceof RegionServerServices;
- rsServices = (RegionServerServices) this.env.getCoprocessorRegionServerServices();
+ rsServices = ((HasRegionServerServices)this.env).getRegionServerServices();
LOG.warn("SecureBulkLoadEndpoint is deprecated. It will be removed in future releases.");
LOG.warn("Secure bulk load has been integrated into HBase core.");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
index ae49253..6a3c1e5 100644
--- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
+++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -37,6 +37,8 @@ import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.constraint.ConstraintException;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.HasMasterServices;
import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.MasterObserver;
@@ -72,6 +74,7 @@ import org.apache.hadoop.hbase.shaded.com.google.common.collect.Sets;
import org.apache.yetus.audience.InterfaceAudience;
// TODO: Encapsulate MasterObserver functions into separate subclass.
+@CoreCoprocessor
@InterfaceAudience.Private
public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
private static final Log LOG = LogFactory.getLog(RSGroupAdminEndpoint.class);
@@ -85,7 +88,10 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
@Override
public void start(CoprocessorEnvironment env) throws IOException {
- master = ((MasterCoprocessorEnvironment)env).getMasterServices();
+ if (!(env instanceof HasMasterServices)) {
+ throw new IOException("Does not implement HMasterServices");
+ }
+ master = ((HasMasterServices)env).getMasterServices();
groupInfoManager = RSGroupInfoManagerImpl.getInstance(master);
groupAdminServer = new RSGroupAdminServer(master, groupInfoManager);
Class<?> clazz =
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java
index f9cb3be..f436dcc1a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
@@ -26,8 +27,9 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.yetus.audience.InterfaceAudience;
/**
- * Defines the set of shared functions implemented by HBase servers (Masters
- * and RegionServers).
+ * Defines a curated set of shared functions implemented by HBase servers (Masters
+ * and RegionServers). For use internally only. Be judicious adding API. Changes cause ripples
+ * through the code base.
*/
@InterfaceAudience.Private
public interface Server extends Abortable, Stoppable {
@@ -79,4 +81,17 @@ public interface Server extends Abortable, Stoppable {
* @return The {@link ChoreService} instance for this server
*/
ChoreService getChoreService();
+
+ /**
+ * @return Return the FileSystem object used.
+ */
+ // TODO: On Master, return Master's. On RegionServer, return RegionServers. The FileSystems
+ // may differ. TODO.
+ FileSystem getFileSystem();
+
+ /**
+ * @return True is the server is Stopping
+ */
+ // Note: This method is not part of the Stoppable Interface.
+ boolean isStopping();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseEnvironment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseEnvironment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseEnvironment.java
index c71c3c4..32cef9e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseEnvironment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseEnvironment.java
@@ -51,8 +51,7 @@ public class BaseEnvironment<C extends Coprocessor> implements CoprocessorEnviro
* @param impl the coprocessor instance
* @param priority chaining priority
*/
- public BaseEnvironment(final C impl, final int priority,
- final int seq, final Configuration conf) {
+ public BaseEnvironment(final C impl, final int priority, final int seq, final Configuration conf) {
this.impl = impl;
this.classLoader = impl.getClass().getClassLoader();
this.priority = priority;
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoreCoprocessor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoreCoprocessor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoreCoprocessor.java
new file mode 100644
index 0000000..0eb5e15
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoreCoprocessor.java
@@ -0,0 +1,45 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import org.apache.yetus.audience.InterfaceAudience;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker annotation that denotes Coprocessors that are core to HBase.
+ * A Core Coprocessor is a CP that realizes a core HBase feature. Features are sometimes
+ * implemented first as a Coprocessor to prove viability. The idea is that once proven, they then
+ * migrate to core. Meantime, HBase Core Coprocessors get this annotation. No other Coprocessors
+ * can carry this annotation.
+ */
+// Core Coprocessors are generally naughty making use of HBase internals doing accesses no
+// Coprocessor should be up to so we mark these special Coprocessors with this annotation and on
+// Coprocessor load, we'll give these Coprocessors a 'richer' Environment with access to internals
+// not allowed other Coprocessors. see the *CoprocessorHost where they do the Coprocessor loadings.
+@Target(ElementType.TYPE)
+@Inherited
+@InterfaceAudience.Private
+@Retention(RetentionPolicy.RUNTIME)
+// This Annotation is not @Documented because I don't want users figuring out its mechanics.
+public @interface CoreCoprocessor {}
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/HasMasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/HasMasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/HasMasterServices.java
new file mode 100644
index 0000000..595e2d7
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/HasMasterServices.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Mark a class that it has a MasterServices accessor.
+ * Temporary hack until core Coprocesssors are integrated.
+ * @see CoreCoprocessor
+ * @deprecated Since 2.0.0 to be removed in 3.0.0. The hope is that by 3.0.0 we will not need this
+ * facility as CoreCoprocessors are integated into core.
+ */
+@Deprecated
+@InterfaceAudience.Private
+public interface HasMasterServices {
+ /**
+ * @return An instance of RegionServerServices, an object NOT for Coprocessor consumption.
+ */
+ MasterServices getMasterServices();
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/HasRegionServerServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/HasRegionServerServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/HasRegionServerServices.java
new file mode 100644
index 0000000..1c060cc
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/HasRegionServerServices.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Mark a class that it has a RegionServiceServices accessor.
+ * Temporary hack until core Coprocesssors are integrated.
+ * @see CoreCoprocessor
+ * @deprecated Since 2.0.0 to be removed in 3.0.0. The hope is that by 3.0.0 we will not need this
+ * facility as CoreCoprocessors are integated into core.
+ */
+@Deprecated
+@InterfaceAudience.Private
+public interface HasRegionServerServices {
+ /**
+ * @return An instance of RegionServerServices, an object NOT for Coprocessor consumption.
+ */
+ RegionServerServices getRegionServerServices();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java
index 1668b69..ab0eb6b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java
@@ -19,18 +19,36 @@
package org.apache.hadoop.hbase.coprocessor;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
-import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.metrics.MetricRegistry;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
@InterfaceStability.Evolving
public interface MasterCoprocessorEnvironment extends CoprocessorEnvironment<MasterCoprocessor> {
- /** @return reference to the HMaster services */
- MasterServices getMasterServices();
+ /**
+ * @return Hosting Server's ServerName
+ */
+ ServerName getServerName();
+
+ /**
+ * Be careful RPC'ing from a Coprocessor context.
+ * RPC's will fail, stall, retry, and/or crawl because the remote side is not online, is
+ * struggling or it is on the other side of a network partition. Any use of Connection from
+ * inside a Coprocessor must be able to handle all such hiccups.
+ *
+ * <p>Using a Connection to get at a local resource -- say a Region that is on the local
+ * Server or using Admin Interface from a Coprocessor hosted on the Master -- will result in a
+ * short-circuit of the RPC framework to make a direct invocation avoiding RPC (and
+ * protobuf marshalling/unmarshalling).
+ *
+ * @return The host's Connection to the Cluster.
+ */
+ Connection getConnection();
/**
* Returns a MetricRegistry that can be used to track metrics at the master level.
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java
index b29cd28..8058a9d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java
@@ -23,9 +23,10 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.metrics.MetricRegistry;
-import org.apache.hadoop.hbase.regionserver.CoprocessorRegionServerServices;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
@@ -39,13 +40,30 @@ public interface RegionCoprocessorEnvironment extends CoprocessorEnvironment<Reg
/** @return region information for the region this coprocessor is running on */
RegionInfo getRegionInfo();
- /** @return reference to the region server services */
- CoprocessorRegionServerServices getCoprocessorRegionServerServices();
-
/** @return shared data between all instances of this coprocessor */
ConcurrentMap<String, Object> getSharedData();
/**
+ * @return Hosting Server's ServerName
+ */
+ ServerName getServerName();
+
+ /**
+ * Be careful RPC'ing from a Coprocessor context.
+ * RPC's will fail, stall, retry, and/or crawl because the remote side is not online, is
+ * struggling or it is on the other side of a network partition. Any use of Connection from
+ * inside a Coprocessor must be able to handle all such hiccups.
+ *
+ * <p>Using a Connection to get at a local resource -- say a Region that is on the local
+ * Server or using Admin Interface from a Coprocessor hosted on the Master -- will result in a
+ * short-circuit of the RPC framework to make a direct invocation avoiding RPC (and
+ * protobuf marshalling/unmarshalling).
+ *
+ * @return The host's Connection to the Cluster.
+ */
+ Connection getConnection();
+
+ /**
* Returns a MetricRegistry that can be used to track metrics at the region server level. All
* metrics tracked at this level will be shared by all the coprocessor instances
* of the same class in the same region server process. Note that there will be one
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java
index ecd0f3e..d0a728e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -20,8 +20,9 @@ package org.apache.hadoop.hbase.coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.metrics.MetricRegistry;
-import org.apache.hadoop.hbase.regionserver.CoprocessorRegionServerServices;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
@@ -30,11 +31,24 @@ import org.apache.yetus.audience.InterfaceStability;
public interface RegionServerCoprocessorEnvironment
extends CoprocessorEnvironment<RegionServerCoprocessor> {
/**
- * Gets the region server services.
+ * @return Hosting Server's ServerName
+ */
+ ServerName getServerName();
+
+ /**
+ * Be careful RPC'ing from a Coprocessor context.
+ * RPC's will fail, stall, retry, and/or crawl because the remote side is not online, is
+ * struggling or it is on the other side of a network partition. Any use of Connection from
+ * inside a Coprocessor must be able to handle all such hiccups.
+ *
+ * <p>Using a Connection to get at a local resource -- say a Region that is on the local
+ * Server or using Admin Interface from a Coprocessor hosted on the Master -- will result in a
+ * short-circuit of the RPC framework to make a direct invocation avoiding RPC (and
+ * protobuf marshalling/unmarshalling).
*
- * @return the region server services
+ * @return The host's Connection to the Cluster.
*/
- CoprocessorRegionServerServices getCoprocessorRegionServerServices();
+ Connection getConnection();
/**
* Returns a MetricRegistry that can be used to track metrics at the region server level.
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index 24cf166..b1abb77 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -24,7 +24,6 @@ import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
-import java.nio.channels.GatheringByteChannel;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.util.ArrayList;
@@ -584,9 +583,8 @@ public abstract class RpcServer implements RpcServerInterface,
}
/**
- * Helper for {@link #channelRead(java.nio.channels.ReadableByteChannel, java.nio.ByteBuffer)}
- * and {@link #channelWrite(GatheringByteChannel, BufferChain)}. Only
- * one of readCh or writeCh should be non-null.
+ * Helper for {@link #channelRead(java.nio.channels.ReadableByteChannel, java.nio.ByteBuffer).
+ * Only one of readCh or writeCh should be non-null.
*
* @param readCh read channel
* @param writeCh write channel
@@ -594,7 +592,6 @@ public abstract class RpcServer implements RpcServerInterface,
* @return bytes written
* @throws java.io.IOException e
* @see #channelRead(java.nio.channels.ReadableByteChannel, java.nio.ByteBuffer)
- * @see #channelWrite(GatheringByteChannel, BufferChain)
*/
private static int channelIO(ReadableByteChannel readCh,
WritableByteChannel writeCh,
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index f4e89b5..70e0891 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.RegionInfo;
@@ -41,6 +42,8 @@ import org.apache.hadoop.hbase.coprocessor.BaseEnvironment;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
import org.apache.hadoop.hbase.coprocessor.CoprocessorServiceBackwardCompatiblity;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.HasMasterServices;
import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.MasterObserver;
@@ -76,14 +79,16 @@ public class MasterCoprocessorHost
*/
private static class MasterEnvironment extends BaseEnvironment<MasterCoprocessor>
implements MasterCoprocessorEnvironment {
- private final MasterServices masterServices;
+ private final Connection connection;
+ private final ServerName serverName;
private final boolean supportGroupCPs;
private final MetricRegistry metricRegistry;
public MasterEnvironment(final MasterCoprocessor impl, final int priority, final int seq,
final Configuration conf, final MasterServices services) {
super(impl, priority, seq, conf);
- this.masterServices = services;
+ this.connection = services.getConnection();
+ this.serverName = services.getServerName();
supportGroupCPs = !useLegacyMethod(impl.getClass(),
"preBalanceRSGroup", ObserverContext.class, String.class);
this.metricRegistry =
@@ -91,8 +96,13 @@ public class MasterCoprocessorHost
}
@Override
- public MasterServices getMasterServices() {
- return masterServices;
+ public ServerName getServerName() {
+ return this.serverName;
+ }
+
+ @Override
+ public Connection getConnection() {
+ return this.connection;
}
@Override
@@ -107,6 +117,29 @@ public class MasterCoprocessorHost
}
}
+ /**
+ * Special version of MasterEnvironment that exposes MasterServices for Core Coprocessors only.
+ * Temporary hack until Core Coprocessors are integrated into Core.
+ */
+ private static class MasterEnvironmentForCoreCoprocessors extends MasterEnvironment
+ implements HasMasterServices {
+ private final MasterServices masterServices;
+
+ public MasterEnvironmentForCoreCoprocessors(final MasterCoprocessor impl, final int priority,
+ final int seq, final Configuration conf, final MasterServices services) {
+ super(impl, priority, seq, conf, services);
+ this.masterServices = services;
+ }
+
+ /**
+ * @return An instance of MasterServices, an object NOT for general user-space Coprocessor
+ * consumption.
+ */
+ public MasterServices getMasterServices() {
+ return this.masterServices;
+ }
+ }
+
private MasterServices masterServices;
public MasterCoprocessorHost(final MasterServices services, final Configuration conf) {
@@ -122,8 +155,6 @@ public class MasterCoprocessorHost
loadSystemCoprocessors(conf, MASTER_COPROCESSOR_CONF_KEY);
}
-
-
@Override
public MasterEnvironment createEnvironment(final MasterCoprocessor instance, final int priority,
final int seq, final Configuration conf) {
@@ -131,7 +162,10 @@ public class MasterCoprocessorHost
for (Service service : instance.getServices()) {
masterServices.registerService(service);
}
- return new MasterEnvironment(instance, priority, seq, conf, masterServices);
+ // If a CoreCoprocessor, return a 'richer' environment, one laden with MasterServices.
+ return instance.getClass().isAnnotationPresent(CoreCoprocessor.class)?
+ new MasterEnvironmentForCoreCoprocessors(instance, priority, seq, conf, masterServices):
+ new MasterEnvironment(instance, priority, seq, conf, masterServices);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index 055a480..57f7df5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -55,7 +55,10 @@ import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTe
import com.google.protobuf.Service;
/**
- * Services Master supplies
+ * A curated subset of services provided by {@link HMaster}.
+ * For use internally only. Passed to Managers, Services and Chores so can pass less-than-a
+ * full-on HMaster at test-time. Be judicious adding API. Changes cause ripples through
+ * the code base.
*/
@InterfaceAudience.Private
public interface MasterServices extends Server {
@@ -414,11 +417,6 @@ public interface MasterServices extends Server {
*/
public LoadBalancer getLoadBalancer();
- /**
- * @return True if this master is stopping.
- */
- boolean isStopping();
-
boolean isSplitOrMergeEnabled(MasterSwitchType switchType);
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
index d6dbcd4..9983875 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.MasterObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
/**
@@ -64,9 +63,8 @@ public class MasterSpaceQuotaObserver implements MasterCoprocessor, MasterObserv
if (!quotasEnabled) {
return;
}
- final MasterServices master = ctx.getEnvironment().getMasterServices();
- final Connection conn = master.getConnection();
- Quotas quotas = QuotaUtil.getTableQuota(master.getConnection(), tableName);
+ final Connection conn = ctx.getEnvironment().getConnection();
+ Quotas quotas = QuotaUtil.getTableQuota(conn, tableName);
if (quotas != null && quotas.hasSpace()) {
QuotaSettings settings = QuotaSettingsFactory.removeTableSpaceLimit(tableName);
try (Admin admin = conn.getAdmin()) {
@@ -82,9 +80,8 @@ public class MasterSpaceQuotaObserver implements MasterCoprocessor, MasterObserv
if (!quotasEnabled) {
return;
}
- final MasterServices master = ctx.getEnvironment().getMasterServices();
- final Connection conn = master.getConnection();
- Quotas quotas = QuotaUtil.getNamespaceQuota(master.getConnection(), namespace);
+ final Connection conn = ctx.getEnvironment().getConnection();
+ Quotas quotas = QuotaUtil.getNamespaceQuota(conn, namespace);
if (quotas != null && quotas.hasSpace()) {
QuotaSettings settings = QuotaSettingsFactory.removeNamespaceSpaceLimit(namespace);
try (Admin admin = conn.getAdmin()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
index e429598..6a5e38c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.quotas;
import static org.apache.hadoop.hbase.util.CollectionUtils.computeIfAbsent;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
@@ -188,7 +189,7 @@ public class QuotaCache implements Stoppable {
justification="I do not understand why the complaints, it looks good to me -- FIX")
protected void chore() {
// Prefetch online tables/namespaces
- for (TableName table: QuotaCache.this.rsServices.getOnlineTables()) {
+ for (TableName table: ((HRegionServer)QuotaCache.this.rsServices).getOnlineTables()) {
if (table.isSystemTable()) continue;
if (!QuotaCache.this.tableQuotaCache.containsKey(table)) {
QuotaCache.this.tableQuotaCache.putIfAbsent(table, new QuotaState());
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CoprocessorRegionServerServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CoprocessorRegionServerServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CoprocessorRegionServerServices.java
deleted file mode 100644
index 3d70793..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CoprocessorRegionServerServices.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import java.util.Set;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hbase.HBaseInterfaceAudience;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.yetus.audience.InterfaceStability;
-
-/**
- * Services exposed to CPs by {@link HRegionServer}
- */
-@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
-@InterfaceStability.Evolving
-public interface CoprocessorRegionServerServices extends ImmutableOnlineRegions {
-
- /**
- * @return True if this regionserver is stopping.
- */
- boolean isStopping();
-
- /**
- * @return Return the FileSystem object used by the regionserver
- */
- FileSystem getFileSystem();
-
- /**
- * @return all the online tables in this RS
- */
- Set<TableName> getOnlineTables();
-
- /**
- * Returns a reference to the servers' connection.
- *
- * Important note: this method returns a reference to Connection which is managed
- * by Server itself, so callers must NOT attempt to close connection obtained.
- */
- Connection getConnection();
-
- /**
- * @return The unique server name for this server.
- */
- ServerName getServerName();
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index da3a1e9..84b0d6a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 60f4e14..d396b3e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -3008,12 +3008,12 @@ public class HRegionServer extends HasThread implements
}
return allRegions;
}
+
/**
* Gets the online tables in this RS.
* This method looks at the in-memory onlineRegions.
* @return all the online tables in this RS
*/
- @Override
public Set<TableName> getOnlineTables() {
Set<TableName> tables = new HashSet<>();
synchronized (this.onlineRegions) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index fbd93b8..8000a2f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -43,7 +43,9 @@ import org.apache.hadoop.hbase.CompareOperator;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
@@ -59,7 +61,9 @@ import org.apache.hadoop.hbase.coprocessor.BulkLoadObserver;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
import org.apache.hadoop.hbase.coprocessor.CoprocessorServiceBackwardCompatiblity;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
import org.apache.hadoop.hbase.coprocessor.EndpointObserver;
+import org.apache.hadoop.hbase.coprocessor.HasRegionServerServices;
import org.apache.hadoop.hbase.coprocessor.MetricsCoprocessor;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
@@ -107,13 +111,13 @@ public class RegionCoprocessorHost
*
* Encapsulation of the environment of each coprocessor
*/
- static class RegionEnvironment extends BaseEnvironment<RegionCoprocessor>
+ private static class RegionEnvironment extends BaseEnvironment<RegionCoprocessor>
implements RegionCoprocessorEnvironment {
-
private Region region;
- private RegionServerServices rsServices;
ConcurrentMap<String, Object> sharedData;
private final MetricRegistry metricRegistry;
+ private final Connection connection;
+ private final ServerName serverName;
/**
* Constructor
@@ -125,7 +129,9 @@ public class RegionCoprocessorHost
final RegionServerServices services, final ConcurrentMap<String, Object> sharedData) {
super(impl, priority, seq, conf);
this.region = region;
- this.rsServices = services;
+ // Mocks may have services as null at test time.
+ this.connection = services != null? services.getConnection(): null;
+ this.serverName = services != null? services.getServerName(): null;
this.sharedData = sharedData;
this.metricRegistry =
MetricsCoprocessor.createRegistryForRegionCoprocessor(impl.getClass().getName());
@@ -137,10 +143,14 @@ public class RegionCoprocessorHost
return region;
}
- /** @return reference to the region server services */
@Override
- public CoprocessorRegionServerServices getCoprocessorRegionServerServices() {
- return rsServices;
+ public Connection getConnection() {
+ return this.connection;
+ }
+
+ @Override
+ public ServerName getServerName() {
+ return this.serverName;
}
@Override
@@ -165,6 +175,30 @@ public class RegionCoprocessorHost
}
}
+ /**
+ * Special version of RegionEnvironment that exposes RegionServerServices for Core
+ * Coprocessors only. Temporary hack until Core Coprocessors are integrated into Core.
+ */
+ private static class RegionEnvironmentForCoreCoprocessors extends
+ RegionEnvironment implements HasRegionServerServices {
+ private final RegionServerServices rsServices;
+
+ public RegionEnvironmentForCoreCoprocessors(final RegionCoprocessor impl, final int priority,
+ final int seq, final Configuration conf, final Region region,
+ final RegionServerServices services, final ConcurrentMap<String, Object> sharedData) {
+ super(impl, priority, seq, conf, region, services, sharedData);
+ this.rsServices = services;
+ }
+
+ /**
+ * @return An instance of RegionServerServices, an object NOT for general user-space Coprocessor
+ * consumption.
+ */
+ public RegionServerServices getRegionServerServices() {
+ return this.rsServices;
+ }
+ }
+
static class TableCoprocessorAttribute {
private Path path;
private String className;
@@ -405,8 +439,11 @@ public class RegionCoprocessorHost
SHARED_DATA_MAP.computeIfAbsent(instance.getClass().getName(),
k -> new ConcurrentHashMap<>());
}
- return new RegionEnvironment(instance, priority, seq, conf, region,
- rsServices, classData);
+ // If a CoreCoprocessor, return a 'richer' environment, one laden with RegionServerServices.
+ return instance.getClass().isAnnotationPresent(CoreCoprocessor.class)?
+ new RegionEnvironmentForCoreCoprocessors(instance, priority, seq, conf, region,
+ rsServices, classData):
+ new RegionEnvironment(instance, priority, seq, conf, region, rsServices, classData);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
index fcf8afb..33be479 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -24,10 +24,13 @@ import com.google.protobuf.Service;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.coprocessor.BaseEnvironment;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.coprocessor.CoprocessorServiceBackwardCompatiblity;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.HasRegionServerServices;
import org.apache.hadoop.hbase.coprocessor.MetricsCoprocessor;
import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessor;
import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
@@ -37,7 +40,6 @@ import org.apache.hadoop.hbase.metrics.MetricRegistry;
import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.security.User;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.yetus.audience.InterfaceStability;
@InterfaceAudience.Private
public class RegionServerCoprocessorHost extends
@@ -68,7 +70,11 @@ public class RegionServerCoprocessorHost extends
@Override
public RegionServerEnvironment createEnvironment(
RegionServerCoprocessor instance, int priority, int sequence, Configuration conf) {
- return new RegionServerEnvironment(instance, priority, sequence, conf, this.rsServices);
+ // If a CoreCoprocessor, return a 'richer' environment, one laden with RegionServerServices.
+ return instance.getClass().isAnnotationPresent(CoreCoprocessor.class)?
+ new RegionServerEnvironmentForCoreCoprocessors(instance, priority, sequence, conf,
+ this.rsServices):
+ new RegionServerEnvironment(instance, priority, sequence, conf, this.rsServices);
}
@Override
@@ -197,26 +203,33 @@ public class RegionServerCoprocessorHost extends
*/
private static class RegionServerEnvironment extends BaseEnvironment<RegionServerCoprocessor>
implements RegionServerCoprocessorEnvironment {
- private final RegionServerServices regionServerServices;
private final MetricRegistry metricRegistry;
+ private final Connection connection;
+ private final ServerName serverName;
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="BC_UNCONFIRMED_CAST",
justification="Intentional; FB has trouble detecting isAssignableFrom")
public RegionServerEnvironment(final RegionServerCoprocessor impl, final int priority,
final int seq, final Configuration conf, final RegionServerServices services) {
super(impl, priority, seq, conf);
- this.regionServerServices = services;
// If coprocessor exposes any services, register them.
for (Service service : impl.getServices()) {
- regionServerServices.registerService(service);
+ services.registerService(service);
}
+ this.connection = services.getConnection();
+ this.serverName = services.getServerName();
this.metricRegistry =
MetricsCoprocessor.createRegistryForRSCoprocessor(impl.getClass().getName());
}
@Override
- public CoprocessorRegionServerServices getCoprocessorRegionServerServices() {
- return regionServerServices;
+ public ServerName getServerName() {
+ return this.serverName;
+ }
+
+ @Override
+ public Connection getConnection() {
+ return this.connection;
}
@Override
@@ -230,4 +243,28 @@ public class RegionServerCoprocessorHost extends
MetricsCoprocessor.removeRegistry(metricRegistry);
}
}
+
+ /**
+ * Special version of RegionServerEnvironment that exposes RegionServerServices for Core
+ * Coprocessors only. Temporary hack until Core Coprocessors are integrated into Core.
+ */
+ private static class RegionServerEnvironmentForCoreCoprocessors extends RegionServerEnvironment
+ implements HasRegionServerServices {
+ final RegionServerServices regionServerServices;
+
+ public RegionServerEnvironmentForCoreCoprocessors(final RegionServerCoprocessor impl,
+ final int priority, final int seq, final Configuration conf,
+ final RegionServerServices services) {
+ super(impl, priority, seq, conf, services);
+ this.regionServerServices = services;
+ }
+
+ /**
+ * @return An instance of RegionServerServices, an object NOT for general user-space Coprocessor
+ * consumption.
+ */
+ public RegionServerServices getRegionServerServices() {
+ return this.regionServerServices;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
index d04f382..d8e8ac5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
@@ -41,11 +41,13 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto
import com.google.protobuf.Service;
/**
- * Services provided by {@link HRegionServer}
+ * A curated subset of services provided by {@link HRegionServer}.
+ * For use internally only. Passed to Managers, Services and Chores so can pass less-than-a
+ * full-on HRegionServer at test-time. Be judicious adding API. Changes cause ripples through
+ * the code base.
*/
@InterfaceAudience.Private
-public interface RegionServerServices
- extends Server, OnlineRegions, FavoredNodesForRegion, CoprocessorRegionServerServices {
+public interface RegionServerServices extends Server, OnlineRegions, FavoredNodesForRegion {
/** @return the WAL for a particular region. Pass null for getting the
* default (common) WAL */
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java
index 32ec617..e72f6e2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java
@@ -27,7 +27,10 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.HasRegionServerServices;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
@@ -37,11 +40,13 @@ import org.apache.hadoop.hbase.coprocessor.RegionObserver;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.util.Pair;
+import javax.validation.constraints.Null;
+
/**
- * An Observer to facilitate replication operations
+ * An Observer to add HFile References to replication queue.
*/
-
-@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+@CoreCoprocessor
+@InterfaceAudience.Private
public class ReplicationObserver implements RegionCoprocessor, RegionObserver {
private static final Log LOG = LogFactory.getLog(ReplicationObserver.class);
@@ -51,19 +56,24 @@ public class ReplicationObserver implements RegionCoprocessor, RegionObserver {
}
@Override
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH",
+ justification="NPE should never happen; if it does it is a bigger issue")
public void preCommitStoreFile(final ObserverContext<RegionCoprocessorEnvironment> ctx,
final byte[] family, final List<Pair<Path, Path>> pairs) throws IOException {
RegionCoprocessorEnvironment env = ctx.getEnvironment();
Configuration c = env.getConfiguration();
- if (pairs == null || pairs.isEmpty()
- || !c.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
+ if (pairs == null || pairs.isEmpty() ||
+ !c.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT)) {
LOG.debug("Skipping recording bulk load entries in preCommitStoreFile for bulkloaded "
+ "data replication.");
return;
}
- HRegionServer rs = (HRegionServer) env.getCoprocessorRegionServerServices();
- Replication rep = (Replication) rs.getReplicationSourceService();
+ // This is completely cheating AND getting a HRegionServer from a RegionServerEnvironment is
+ // just going to break. This is all private. Not allowed. Regions shouldn't assume they are
+ // hosted in a RegionServer. TODO: fix.
+ RegionServerServices rss = ((HasRegionServerServices)env).getRegionServerServices();
+ Replication rep = (Replication)((HRegionServer)rss).getReplicationSourceService();
rep.addHFileRefsToQueue(env.getRegionInfo().getTable(), family, pairs);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
index 7fdb252..280289c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -193,5 +193,15 @@ public class ReplicationSyncUp extends Configured implements Tool {
// TODO Auto-generated method stub
return null;
}
+
+ @Override
+ public FileSystem getFileSystem() {
+ return null;
+ }
+
+ @Override
+ public boolean isStopping() {
+ return false;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
index 9cdec3c..6c2eb60 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.security.access;
import org.apache.hadoop.hbase.CompareOperator;
+import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.shaded.com.google.common.collect.ArrayListMultimap;
import org.apache.hadoop.hbase.shaded.com.google.common.collect.ListMultimap;
import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
@@ -43,9 +44,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.AuthUtil;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Tag;
@@ -63,13 +61,10 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
-import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
-import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.security.User;
@@ -126,27 +121,6 @@ public class AccessControlLists {
private static final Log LOG = LogFactory.getLog(AccessControlLists.class);
/**
- * Create the ACL table
- * @param master
- * @throws IOException
- */
- static void createACLTable(MasterServices master) throws IOException {
- /** Table descriptor for ACL table */
- final HTableDescriptor ACL_TABLEDESC = new HTableDescriptor(ACL_TABLE_NAME)
- .addFamily(new HColumnDescriptor(ACL_LIST_FAMILY)
- .setMaxVersions(1)
- .setInMemory(true)
- .setBlockCacheEnabled(true)
- .setBlocksize(8 * 1024)
- .setBloomFilterType(BloomType.NONE)
- .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
- // Set cache data blocks in L1 if more than one cache tier deployed; e.g. this will
- // be the case if we are using CombinedBlockCache (Bucket Cache).
- .setCacheDataInL1(true));
- master.createSystemTable(ACL_TABLEDESC);
- }
-
- /**
* Stores a new user permission grant in the access control lists table.
* @param conf the configuration
* @param userPerm the details of the permission to be granted
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index aa1a765..bfc25a4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -54,13 +54,14 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
-import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
@@ -75,9 +76,13 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.coprocessor.BulkLoadObserver;
import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
+import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor;
import org.apache.hadoop.hbase.coprocessor.EndpointObserver;
+import org.apache.hadoop.hbase.coprocessor.HasMasterServices;
+import org.apache.hadoop.hbase.coprocessor.HasRegionServerServices;
import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.MasterObserver;
@@ -94,7 +99,6 @@ import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
import org.apache.hadoop.hbase.ipc.RpcServer;
-import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.locking.LockProcedure;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.net.Address;
@@ -105,11 +109,11 @@ import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService;
import org.apache.hadoop.hbase.quotas.GlobalQuotaSettings;
+import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.hadoop.hbase.regionserver.Store;
@@ -173,6 +177,7 @@ import org.apache.hadoop.hbase.shaded.com.google.common.collect.Sets;
* commands.
* </p>
*/
+@CoreCoprocessor
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
public class AccessController implements MasterCoprocessor, RegionCoprocessor,
RegionServerCoprocessor, AccessControlService.Interface,
@@ -285,8 +290,7 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
// to and fro conversion overhead. get req is converted to PB req
// and results are converted to PB results 1st and then to POJOs
// again. We could have avoided such at least in ACL table context..
- try (Table t = e.getCoprocessorRegionServerServices().getConnection().
- getTable(AccessControlLists.ACL_TABLE_NAME)) {
+ try (Table t = e.getConnection().getTable(AccessControlLists.ACL_TABLE_NAME)) {
for (byte[] entry : entries) {
currentEntry = entry;
ListMultimap<String, TablePermission> perms =
@@ -955,20 +959,24 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
ZooKeeperWatcher zk = null;
if (env instanceof MasterCoprocessorEnvironment) {
// if running on HMaster
- MasterCoprocessorEnvironment mEnv = (MasterCoprocessorEnvironment) env;
- zk = mEnv.getMasterServices().getZooKeeper();
+ MasterCoprocessorEnvironment mEnv = (MasterCoprocessorEnvironment)env;
+ if (mEnv instanceof HasMasterServices) {
+ zk = ((HasMasterServices)mEnv).getMasterServices().getZooKeeper();
+ }
} else if (env instanceof RegionServerCoprocessorEnvironment) {
- RegionServerCoprocessorEnvironment rsEnv = (RegionServerCoprocessorEnvironment) env;
- assert rsEnv.getCoprocessorRegionServerServices() instanceof RegionServerServices;
- zk = ((RegionServerServices) rsEnv.getCoprocessorRegionServerServices()).getZooKeeper();
+ RegionServerCoprocessorEnvironment rsEnv = (RegionServerCoprocessorEnvironment)env;
+ if (rsEnv instanceof HasRegionServerServices) {
+ zk = ((HasRegionServerServices)rsEnv).getRegionServerServices().getZooKeeper();
+ }
} else if (env instanceof RegionCoprocessorEnvironment) {
// if running at region
regionEnv = (RegionCoprocessorEnvironment) env;
conf.addBytesMap(regionEnv.getRegion().getTableDescriptor().getValues());
- assert regionEnv.getCoprocessorRegionServerServices() instanceof RegionServerServices;
- zk = ((RegionServerServices) regionEnv.getCoprocessorRegionServerServices()).getZooKeeper();
compatibleEarlyTermination = conf.getBoolean(AccessControlConstants.CF_ATTRIBUTE_EARLY_OUT,
- AccessControlConstants.DEFAULT_ATTRIBUTE_EARLY_OUT);
+ AccessControlConstants.DEFAULT_ATTRIBUTE_EARLY_OUT);
+ if (regionEnv instanceof HasRegionServerServices) {
+ zk = ((HasRegionServerServices)regionEnv).getRegionServerServices().getZooKeeper();
+ }
}
// set the user-provider.
@@ -1080,7 +1088,7 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- try (Table table = c.getEnvironment().getMasterServices().getConnection().
+ try (Table table = c.getEnvironment().getConnection().
getTable(AccessControlLists.ACL_TABLE_NAME)) {
AccessControlLists.addUserPermission(c.getEnvironment().getConfiguration(),
userperm, table);
@@ -1106,7 +1114,7 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- try (Table table = c.getEnvironment().getMasterServices().getConnection().
+ try (Table table = c.getEnvironment().getConnection().
getTable(AccessControlLists.ACL_TABLE_NAME)) {
AccessControlLists.removeTablePermissions(conf, tableName, table);
}
@@ -1145,7 +1153,7 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
List<UserPermission> perms = tableAcls.get(tableName);
if (perms != null) {
for (UserPermission perm : perms) {
- try (Table table = ctx.getEnvironment().getMasterServices().getConnection().
+ try (Table table = ctx.getEnvironment().getConnection().
getTable(AccessControlLists.ACL_TABLE_NAME)) {
AccessControlLists.addUserPermission(conf, perm, table);
}
@@ -1176,7 +1184,7 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
public Void run() throws Exception {
UserPermission userperm = new UserPermission(Bytes.toBytes(owner),
htd.getTableName(), null, Action.values());
- try (Table table = c.getEnvironment().getMasterServices().getConnection().
+ try (Table table = c.getEnvironment().getConnection().
getTable(AccessControlLists.ACL_TABLE_NAME)) {
AccessControlLists.addUserPermission(conf, userperm, table);
}
@@ -1215,7 +1223,7 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- try (Table table = ctx.getEnvironment().getMasterServices().getConnection().
+ try (Table table = ctx.getEnvironment().getConnection().
getTable(AccessControlLists.ACL_TABLE_NAME)) {
AccessControlLists.removeTablePermissions(conf, tableName, columnFamily, table);
}
@@ -1370,14 +1378,36 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
@Override
public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
- if (!MetaTableAccessor.tableExists(ctx.getEnvironment().getMasterServices()
- .getConnection(), AccessControlLists.ACL_TABLE_NAME)) {
- // initialize the ACL storage table
- AccessControlLists.createACLTable(ctx.getEnvironment().getMasterServices());
- } else {
- aclTabAvailable = true;
+ try (Admin admin = ctx.getEnvironment().getConnection().getAdmin()) {
+ if (!admin.tableExists(AccessControlLists.ACL_TABLE_NAME)) {
+ createACLTable(admin);
+ } else {
+ this.aclTabAvailable = true;
+ }
}
}
+ /**
+ * Create the ACL table
+ * @throws IOException
+ */
+ private static void createACLTable(Admin admin) throws IOException {
+ /** Table descriptor for ACL table */
+ ColumnFamilyDescriptor cfd =
+ ColumnFamilyDescriptorBuilder.newBuilder(AccessControlLists.ACL_LIST_FAMILY).
+ setMaxVersions(1).
+ setInMemory(true).
+ setBlockCacheEnabled(true).
+ setBlocksize(8 * 1024).
+ setBloomFilterType(BloomType.NONE).
+ setScope(HConstants.REPLICATION_SCOPE_LOCAL).
+ // Set cache data blocks in L1 if more than one cache tier deployed; e.g. this will
+ // be the case if we are using CombinedBlockCache (Bucket Cache).
+ setCacheDataInL1(true).build();
+ TableDescriptor td =
+ TableDescriptorBuilder.newBuilder(AccessControlLists.ACL_TABLE_NAME).
+ addColumnFamily(cfd).build();
+ admin.createTable(td);
+ }
@Override
public void preSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
@@ -1463,7 +1493,7 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- try (Table table = ctx.getEnvironment().getMasterServices().getConnection().
+ try (Table table = ctx.getEnvironment().getConnection().
getTable(AccessControlLists.ACL_TABLE_NAME)) {
AccessControlLists.removeNamespacePermissions(conf, namespace, table);
}
@@ -2309,7 +2339,7 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
@Override
public Void run() throws Exception {
// regionEnv is set at #start. Hopefully not null at this point.
- try (Table table = regionEnv.getCoprocessorRegionServerServices().getConnection().
+ try (Table table = regionEnv.getConnection().
getTable(AccessControlLists.ACL_TABLE_NAME)) {
AccessControlLists.addUserPermission(regionEnv.getConfiguration(), perm, table,
request.getMergeExistingPermissions());
@@ -2366,7 +2396,7 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
@Override
public Void run() throws Exception {
// regionEnv is set at #start. Hopefully not null here.
- try (Table table = regionEnv.getCoprocessorRegionServerServices().getConnection().
+ try (Table table = regionEnv.getConnection().
getTable(AccessControlLists.ACL_TABLE_NAME)) {
AccessControlLists.removeUserPermission(regionEnv.getConfiguration(), perm, table);
}
@@ -2593,13 +2623,16 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
if (regex == null && tableNamesList != null && !tableNamesList.isEmpty()) {
// Otherwise, if the requestor has ADMIN or CREATE privs for all listed tables, the
// request can be granted.
- MasterServices masterServices = ctx.getEnvironment().getMasterServices();
- for (TableName tableName: tableNamesList) {
- // Skip checks for a table that does not exist
- if (!masterServices.getTableStateManager().isTablePresent(tableName))
- continue;
- requirePermission(getActiveUser(ctx), "getTableDescriptors", tableName, null, null,
+ TableName [] sns = null;
+ try (Admin admin = ctx.getEnvironment().getConnection().getAdmin()) {
+ sns = admin.listTableNames();
+ if (sns == null) return;
+ for (TableName tableName: tableNamesList) {
+ // Skip checks for a table that does not exist
+ if (!admin.tableExists(tableName)) continue;
+ requirePermission(getActiveUser(ctx), "getTableDescriptors", tableName, null, null,
Action.ADMIN, Action.CREATE);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/38879fb3/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/CoprocessorWhitelistMasterObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/CoprocessorWhitelistMasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/CoprocessorWhitelistMasterObserver.java
index 5b4acbe..a1179b1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/CoprocessorWhitelistMasterObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/CoprocessorWhitelistMasterObserver.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.MasterObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.yetus.audience.InterfaceAudience;
@@ -147,8 +146,7 @@ public class CoprocessorWhitelistMasterObserver implements MasterCoprocessor, Ma
private void verifyCoprocessors(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableDescriptor htd) throws IOException {
- MasterServices services = ctx.getEnvironment().getMasterServices();
- Configuration conf = services.getConfiguration();
+ Configuration conf = ctx.getEnvironment().getConfiguration();
Collection<String> paths =
conf.getStringCollection(