You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by el...@apache.org on 2017/05/22 20:07:00 UTC
[04/49] hbase git commit: HBASE-17025 Add shell commands for space
quotas
HBASE-17025 Add shell commands for space quotas
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f1066cd7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f1066cd7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f1066cd7
Branch: refs/heads/master
Commit: f1066cd7744e185044dcc33aae09623d10ed2631
Parents: 6c9082f
Author: Josh Elser <el...@apache.org>
Authored: Wed Jan 11 11:55:29 2017 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Mon May 22 13:41:35 2017 -0400
----------------------------------------------------------------------
hbase-shell/src/main/ruby/hbase/quotas.rb | 62 ++++++++-
hbase-shell/src/main/ruby/hbase_constants.rb | 1 +
.../src/main/ruby/shell/commands/set_quota.rb | 45 +++++-
.../hadoop/hbase/client/AbstractTestShell.java | 1 +
hbase-shell/src/test/ruby/hbase/quotas_test.rb | 137 +++++++++++++++++++
hbase-shell/src/test/ruby/tests_runner.rb | 1 +
6 files changed, 242 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1066cd7/hbase-shell/src/main/ruby/hbase/quotas.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb b/hbase-shell/src/main/ruby/hbase/quotas.rb
index bf2dc63..d99fe72 100644
--- a/hbase-shell/src/main/ruby/hbase/quotas.rb
+++ b/hbase-shell/src/main/ruby/hbase/quotas.rb
@@ -24,14 +24,22 @@ java_import org.apache.hadoop.hbase.quotas.ThrottleType
java_import org.apache.hadoop.hbase.quotas.QuotaFilter
java_import org.apache.hadoop.hbase.quotas.QuotaRetriever
java_import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory
+java_import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy
module HBaseQuotasConstants
+ # RPC Quota constants
GLOBAL_BYPASS = 'GLOBAL_BYPASS'
THROTTLE_TYPE = 'THROTTLE_TYPE'
THROTTLE = 'THROTTLE'
REQUEST = 'REQUEST'
WRITE = 'WRITE'
READ = 'READ'
+ # Space quota constants
+ SPACE = 'SPACE'
+ NO_INSERTS = 'NO_INSERTS'
+ NO_WRITES = 'NO_WRITES'
+ NO_WRITES_COMPACTIONS = 'NO_WRITES_COMPACTIONS'
+ DISABLE = 'DISABLE'
end
module Hbase
@@ -107,6 +115,54 @@ module Hbase
@admin.setQuota(settings)
end
+ def limit_space(args)
+ raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil? and args.kind_of?(Hash))
+ # Let the user provide a raw number
+ if args[LIMIT].is_a?(Numeric)
+ limit = args[LIMIT]
+ else
+ # Parse a string a 1K, 2G, etc.
+ limit = _parse_size(args[LIMIT])
+ end
+ # Extract the policy, failing if something bogus was provided
+ policy = SpaceViolationPolicy.valueOf(args[POLICY])
+ # Create a table or namespace quota
+ if args.key?(TABLE)
+ if args.key?(NAMESPACE)
+ raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.")
+ end
+ settings = QuotaSettingsFactory.limitTableSpace(TableName.valueOf(args.delete(TABLE)), limit, policy)
+ elsif args.key?(NAMESPACE)
+ if args.key?(TABLE)
+ raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.")
+ end
+ settings = QuotaSettingsFactory.limitNamespaceSpace(args.delete(NAMESPACE), limit, policy)
+ else
+ raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.')
+ end
+ # Apply the quota
+ @admin.setQuota(settings)
+ end
+
+ def remove_space_limit(args)
+ raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil? and args.kind_of?(Hash))
+ if args.key?(TABLE)
+ if args.key?(NAMESPACE)
+ raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.")
+ end
+ table = TableName.valueOf(args.delete(TABLE))
+ settings = QuotaSettingsFactory.removeTableSpaceLimit(table)
+ elsif args.key?(NAMESPACE)
+ if args.key?(TABLE)
+ raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.")
+ end
+ settings = QuotaSettingsFactory.removeNamespaceSpaceLimit(args.delete(NAMESPACE))
+ else
+ raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.')
+ end
+ @admin.setQuota(settings)
+ end
+
def set_global_bypass(bypass, args)
raise(ArgumentError, "Arguments should be a Hash") unless args.kind_of?(Hash)
@@ -171,7 +227,7 @@ module Hbase
return _size_from_str(match[1].to_i, match[2])
end
else
- raise "Invalid size limit syntax"
+ raise(ArgumentError, "Invalid size limit syntax")
end
end
@@ -188,7 +244,7 @@ module Hbase
end
if limit <= 0
- raise "Invalid throttle limit, must be greater then 0"
+ raise(ArgumentError, "Invalid throttle limit, must be greater then 0")
end
case match[3]
@@ -200,7 +256,7 @@ module Hbase
return type, limit, time_unit
else
- raise "Invalid throttle limit syntax"
+ raise(ArgumentError, "Invalid throttle limit syntax")
end
end
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1066cd7/hbase-shell/src/main/ruby/hbase_constants.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase_constants.rb b/hbase-shell/src/main/ruby/hbase_constants.rb
index 7d6da9f..fe8e812 100644
--- a/hbase-shell/src/main/ruby/hbase_constants.rb
+++ b/hbase-shell/src/main/ruby/hbase_constants.rb
@@ -86,6 +86,7 @@ module HBaseConstants
RESTORE_ACL = 'RESTORE_ACL'
FORMATTER = 'FORMATTER'
FORMATTER_CLASS = 'FORMATTER_CLASS'
+ POLICY = 'POLICY'
# Load constants from hbase java API
def self.promote_constants(constants)
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1066cd7/hbase-shell/src/main/ruby/shell/commands/set_quota.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/set_quota.rb b/hbase-shell/src/main/ruby/shell/commands/set_quota.rb
index a638b93..06ed0ba 100644
--- a/hbase-shell/src/main/ruby/shell/commands/set_quota.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/set_quota.rb
@@ -52,6 +52,37 @@ For example:
hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => WRITE, USER => 'u1', LIMIT => NONE
hbase> set_quota USER => 'u1', GLOBAL_BYPASS => true
+
+TYPE => SPACE
+Users can either set a quota on a table or a namespace. The quota is a limit on the target's
+size on the FileSystem and some action to take when the target exceeds that limit. The limit
+is in bytes and can expressed using standard metric suffixes (B, K, M, G, T, P), defaulting
+to bytes if not provided. Different quotas can be applied to one table at the table and namespace
+level; table-level quotas take priority over namespace-level quotas.
+
+There are a limited number of policies to take when a quota is violation, listed in order of
+least strict to most strict.
+
+ NO_INSERTS - No new data is allowed to be ingested (e.g. Put, Increment, Append).
+ NO_WRITES - Same as NO_INSERTS but Deletes are also disallowed.
+ NO_WRITES_COMPACTIONS - Same as NO_WRITES but compactions are also disallowed.
+ DISABLE - The table(s) are disabled.
+
+For example:
+
+ hbase> set_quota TYPE => SPACE, TABLE => 't1', LIMIT => '1G', POLICY => NO_INSERTS
+ hbase> set_quota TYPE => SPACE, TABLE => 't2', LIMIT => '50G', POLICY => DISABLE
+ hbase> set_quota TYPE => SPACE, TABLE => 't3', LIMIT => '2T', POLICY => NO_WRITES_COMPACTIONS
+ hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => '50T', POLICY => NO_WRITES
+
+Space quotas can also be removed via this command. To remove a space quota, provide NONE
+for the limit.
+
+For example:
+
+ hbase> set_quota TYPE => SPACE, TABLE => 't1', LIMIT => NONE
+ hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => NONE
+
EOF
end
@@ -66,8 +97,18 @@ EOF
else
quotas_admin.throttle(args)
end
- else
- raise "Invalid TYPE argument. got " + qtype
+ when SPACE
+ if args[LIMIT].eql? NONE
+ args.delete(LIMIT)
+ # Table/Namespace argument is verified in remove_space_limit
+ quotas_admin.remove_space_limit(args)
+ else
+ raise(ArgumentError, 'Expected a LIMIT to be provided') unless args.key?(LIMIT)
+ raise(ArgumentError, 'Expected a POLICY to be provided') unless args.key?(POLICY)
+ quotas_admin.limit_space(args)
+ end
+ else
+ raise "Invalid TYPE argument. got " + qtype
end
elsif args.has_key?(GLOBAL_BYPASS)
quotas_admin.set_global_bypass(args.delete(GLOBAL_BYPASS), args)
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1066cd7/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java
----------------------------------------------------------------------
diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java
index 1403805..53606e9 100644
--- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java
+++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java
@@ -39,6 +39,7 @@ public abstract class AbstractTestShell {
// Start mini cluster
TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
+ TEST_UTIL.getConfiguration().setBoolean("hbase.quota.enabled", true);
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
TEST_UTIL.getConfiguration().setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, false);
TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1066cd7/hbase-shell/src/test/ruby/hbase/quotas_test.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/test/ruby/hbase/quotas_test.rb b/hbase-shell/src/test/ruby/hbase/quotas_test.rb
new file mode 100644
index 0000000..78c889c
--- /dev/null
+++ b/hbase-shell/src/test/ruby/hbase/quotas_test.rb
@@ -0,0 +1,137 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'shell'
+require 'stringio'
+require 'hbase_constants'
+require 'hbase/hbase'
+require 'hbase/table'
+
+include HBaseConstants
+
+module Hbase
+ class SpaceQuotasTest < Test::Unit::TestCase
+ include TestHelpers
+
+ def setup
+ setup_hbase
+ # Create test table if it does not exist
+ @test_name = "hbase_shell_tests_table"
+ create_test_table(@test_name)
+ end
+
+ def teardown
+ shutdown
+ end
+
+ define_test 'limit_space errors on non Hash argument' do
+ qa = quotas_admin()
+ assert_raise(ArgumentError) do
+ qa.limit_space('foo')
+ end
+ assert_raise(ArgumentError) do
+ qa.limit_space()
+ end
+ end
+
+ define_test 'remove_space_limit errors on non Hash argument' do
+ qa = quotas_admin()
+ assert_raise(ArgumentError) do
+ qa.remove_space_limit('foo')
+ end
+ assert_raise(ArgumentError) do
+ qa.remove_space_limit()
+ end
+ end
+
+ define_test 'set quota with a non-numeric limit fails' do
+ assert_raise(ArgumentError) do
+ command(:set_quota, TYPE => SPACE, LIMIT => 'asdf', POLICY => NO_INSERTS, TABLE => @test_name)
+ end
+ end
+
+ define_test 'set quota without a limit fails' do
+ assert_raise(ArgumentError) do
+ command(:set_quota, TYPE => SPACE, POLICY => NO_INSERTS, TABLE => @test_name)
+ end
+ end
+
+ define_test 'set quota without a policy fails' do
+ assert_raise(ArgumentError) do
+ command(:set_quota, TYPE => SPACE, LIMIT => '1G', TABLE => @test_name)
+ end
+ end
+
+ define_test 'set quota without a table or namespace fails' do
+ assert_raise(ArgumentError) do
+ command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => NO_INSERTS)
+ end
+ end
+
+ define_test 'invalid violation policy specified' do
+ assert_raise(NameError) do
+ command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => FOO_BAR, TABLE => @test_name)
+ end
+ end
+
+ define_test 'table and namespace are mutually exclusive in set quota' do
+ assert_raise(ArgumentError) do
+ command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => NO_INSERTS, TABLE => @test_name, NAMESPACE => "foo")
+ end
+ end
+
+ define_test '_parse_size accepts various forms of byte shorthand' do
+ qa = quotas_admin()
+ KILO = 1024
+ MEGA = KILO * KILO
+ GIGA = MEGA * KILO
+ TERA = GIGA * KILO
+ PETA = TERA * KILO
+ assert_equal(1, qa._parse_size("1"))
+ assert_equal(1, qa._parse_size("1b"))
+ assert_equal(1, qa._parse_size("1B"))
+ assert_equal(KILO * 2, qa._parse_size("2k"))
+ assert_equal(KILO * 2, qa._parse_size("2K"))
+ assert_equal(MEGA * 5, qa._parse_size("5m"))
+ assert_equal(MEGA * 5, qa._parse_size("5M"))
+ assert_equal(GIGA * 3, qa._parse_size("3g"))
+ assert_equal(GIGA * 3, qa._parse_size("3G"))
+ assert_equal(TERA * 4, qa._parse_size("4t"))
+ assert_equal(TERA * 4, qa._parse_size("4T"))
+ assert_equal(PETA * 32, qa._parse_size("32p"))
+ assert_equal(PETA * 32, qa._parse_size("32P"))
+ assert_equal(GIGA * 4, qa._parse_size("4096m"))
+ assert_equal(GIGA * 4, qa._parse_size("4096M"))
+ end
+
+ define_test 'can set and remove quota' do
+ command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => NO_INSERTS, TABLE => @test_name)
+ output = capture_stdout{ command(:list_quotas) }
+ size = 1024 * 1024 * 1024
+ assert(output.include?("LIMIT => #{size}"))
+ assert(output.include?("VIOLATION_POLICY => NO_INSERTS"))
+ assert(output.include?("TYPE => SPACE"))
+ assert(output.include?("TABLE => #{@test_name}"))
+
+ command(:set_quota, TYPE => SPACE, LIMIT => NONE, TABLE => @test_name)
+ output = capture_stdout{ command(:list_quotas) }
+ assert(output.include?("0 row(s)"))
+ end
+ end
+end
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1066cd7/hbase-shell/src/test/ruby/tests_runner.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/test/ruby/tests_runner.rb b/hbase-shell/src/test/ruby/tests_runner.rb
index 74ddb48..54bf3f9 100644
--- a/hbase-shell/src/test/ruby/tests_runner.rb
+++ b/hbase-shell/src/test/ruby/tests_runner.rb
@@ -36,6 +36,7 @@ unless defined?($TEST_CLUSTER)
$TEST_CLUSTER = HBaseTestingUtility.new
$TEST_CLUSTER.configuration.setInt("hbase.regionserver.msginterval", 100)
$TEST_CLUSTER.configuration.setInt("hbase.client.pause", 250)
+ $TEST_CLUSTER.configuration.set("hbase.quota.enabled", "true")
$TEST_CLUSTER.configuration.setInt(org.apache.hadoop.hbase.HConstants::HBASE_CLIENT_RETRIES_NUMBER, 6)
$TEST_CLUSTER.startMiniCluster
@own_cluster = true