You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2014/10/21 20:15:31 UTC
git commit: HBASE-12286 [shell] Add server/cluster online load of
configuration changes Add doc to the refguide,
change command names to drop the _rs_
Repository: hbase
Updated Branches:
refs/heads/master 775326907 -> f61df3398
HBASE-12286 [shell] Add server/cluster online load of configuration changes Add doc to the refguide, change command names to drop the _rs_
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f61df339
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f61df339
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f61df339
Branch: refs/heads/master
Commit: f61df339889384e0848ccf4ba392f7ff5fbd042d
Parents: 7753269
Author: stack <st...@apache.org>
Authored: Tue Oct 21 09:09:43 2014 -0700
Committer: stack <st...@apache.org>
Committed: Tue Oct 21 11:15:09 2014 -0700
----------------------------------------------------------------------
hbase-shell/src/main/ruby/hbase/admin.rb | 81 ++++++++++++--------
hbase-shell/src/main/ruby/shell.rb | 9 +++
.../ruby/shell/commands/update_all_config.rb | 39 ++++++++++
.../main/ruby/shell/commands/update_config.rb | 40 ++++++++++
src/main/docbkx/configuration.xml | 18 +++++
5 files changed, 153 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f61df339/hbase-shell/src/main/ruby/hbase/admin.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb
index a242846..2a5876a 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -22,6 +22,7 @@ java_import java.util.Arrays
java_import org.apache.hadoop.hbase.util.Pair
java_import org.apache.hadoop.hbase.util.RegionSplitter
java_import org.apache.hadoop.hbase.util.Bytes
+java_import org.apache.hadoop.hbase.ServerName
java_import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos::SnapshotDescription
# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin
@@ -211,7 +212,7 @@ module Hbase
unless arg.kind_of?(String) || arg.kind_of?(Hash)
raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type")
end
-
+
# First, handle all the cases where arg is a column family.
if arg.kind_of?(String) or arg.has_key?(NAME)
# If the arg is a string, default action is to add a column to the table.
@@ -223,14 +224,14 @@ module Hbase
if arg.has_key?(REGION_REPLICATION)
region_replication = JInteger.valueOf(arg.delete(REGION_REPLICATION))
htd.setRegionReplication(region_replication)
- end
-
+ end
+
# Get rid of the "METHOD", which is deprecated for create.
# We'll do whatever it used to do below if it's table_att.
if (method = arg.delete(METHOD))
raise(ArgumentError, "table_att is currently the only supported method") unless method == 'table_att'
end
-
+
# The hash is not a column family. Figure out what's in it.
# First, handle splits.
if arg.has_key?(SPLITS_FILE)
@@ -245,7 +246,7 @@ module Hbase
htd.setValue(SPLITS_FILE, arg[SPLITS_FILE])
end
- if arg.has_key?(SPLITS)
+ if arg.has_key?(SPLITS)
splits = Java::byte[][arg[SPLITS].size].new
idx = 0
arg.delete(SPLITS).each do |split|
@@ -261,9 +262,9 @@ module Hbase
split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg.delete(SPLITALGO))
splits = split_algo.split(JInteger.valueOf(num_regions))
end
-
+
# Done with splits; apply formerly-table_att parameters.
- htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER]
+ htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER]
htd.setMaxFileSize(JLong.valueOf(arg.delete(MAX_FILESIZE))) if arg[MAX_FILESIZE]
htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY]
htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED]
@@ -272,15 +273,15 @@ module Hbase
htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(DURABILITY))) if arg[DURABILITY]
set_user_metadata(htd, arg.delete(METADATA)) if arg[METADATA]
set_descriptor_config(htd, arg.delete(CONFIGURATION)) if arg[CONFIGURATION]
-
+
arg.each_key do |ignored_key|
puts("An argument ignored (unknown or overridden): %s" % [ ignored_key ])
end
end
-
+
# Fail if no column families defined
raise(ArgumentError, "Table must have at least one column family") if !has_columns
-
+
if splits.nil?
# Perform the create table call
@admin.createTable(htd)
@@ -289,15 +290,15 @@ module Hbase
@admin.createTable(htd, splits)
end
end
-
+
#----------------------------------------------------------------------------------------------
# Closes a region.
# If server name is nil, we presume region_name is full region name (HRegionInfo.getRegionName).
# If server name is not nil, we presume it is the region's encoded name (HRegionInfo.getEncodedName)
def close_region(region_name, server)
- if (server == nil || !closeEncodedRegion?(region_name, server))
+ if (server == nil || !closeEncodedRegion?(region_name, server))
@admin.closeRegion(region_name, server)
- end
+ end
end
#----------------------------------------------------------------------------------------------
@@ -318,7 +319,7 @@ module Hbase
def move(encoded_region_name, server = nil)
@admin.move(encoded_region_name.to_java_bytes, server ? server.to_java_bytes: nil)
end
-
+
#----------------------------------------------------------------------------------------------
# Merge two regions
def merge_region(encoded_region_a_name, encoded_region_b_name, force)
@@ -433,14 +434,14 @@ module Hbase
# Process all args
args.each do |arg|
-
-
+
+
# Normalize args to support column name only alter specs
arg = { NAME => arg } if arg.kind_of?(String)
# Normalize args to support shortcut delete syntax
arg = { METHOD => 'delete', NAME => arg['delete'] } if arg['delete']
-
+
# There are 3 possible options.
# 1) Column family spec. Distinguished by having a NAME and no METHOD.
method = arg.delete(METHOD)
@@ -459,12 +460,12 @@ module Hbase
puts "Updating all regions with the new schema..."
alter_status(table_name)
end
-
+
# We bypass descriptor when adding column families; refresh it to apply other args correctly.
htd = @admin.getTableDescriptor(table_name.to_java_bytes)
next
end
-
+
# 2) Method other than table_att, with some args.
name = arg.delete(NAME)
if method != nil and method != "table_att"
@@ -484,26 +485,26 @@ module Hbase
else
raise ArgumentError, "Unknown method: #{method}"
end
-
+
arg.each_key do |unknown_key|
puts("Unknown argument ignored: %s" % [unknown_key])
end
-
+
if wait == true
puts "Updating all regions with the new schema..."
alter_status(table_name)
end
-
+
if method == "delete"
# We bypass descriptor when deleting column families; refresh it to apply other args correctly.
htd = @admin.getTableDescriptor(table_name.to_java_bytes)
end
- next
+ next
end
-
+
# 3) Some args for the table, optionally with METHOD => table_att (deprecated)
raise(ArgumentError, "NAME argument in an unexpected place") if name
- htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER]
+ htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER]
htd.setMaxFileSize(JLong.valueOf(arg.delete(MAX_FILESIZE))) if arg[MAX_FILESIZE]
htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY]
htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED]
@@ -544,17 +545,17 @@ module Hbase
valid_coproc_keys << key
end
end
-
+
valid_coproc_keys.each do |key|
arg.delete(key)
end
@admin.modifyTable(table_name.to_java_bytes, htd)
-
+
arg.each_key do |unknown_key|
puts("Unknown argument ignored: %s" % [unknown_key])
end
-
+
if wait == true
puts "Updating all regions with the new schema..."
alter_status(table_name)
@@ -621,7 +622,7 @@ module Hbase
def exists?(table_name)
@admin.tableExists(table_name)
end
-
+
#----------------------------------------------------------------------------------------------
# Is table enabled
def enabled?(table_name)
@@ -632,7 +633,7 @@ module Hbase
#Is supplied region name is encoded region name
def closeEncodedRegion?(region_name, server)
@admin.closeRegionWithEncodedRegionName(region_name, server)
- end
+ end
#----------------------------------------------------------------------------------------------
# Return a new HColumnDescriptor made of passed args
@@ -661,8 +662,8 @@ module Hbase
if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER)
bloomtype = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER).upcase
unless org.apache.hadoop.hbase.regionserver.BloomType.constants.include?(bloomtype)
- raise(ArgumentError, "BloomFilter type #{bloomtype} is not supported. Use one of " + org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(" "))
- else
+ raise(ArgumentError, "BloomFilter type #{bloomtype} is not supported. Use one of " + org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(" "))
+ else
family.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf(bloomtype))
end
end
@@ -691,7 +692,7 @@ module Hbase
arg.each_key do |unknown_key|
puts("Unknown argument ignored for column family %s: %s" % [name, unknown_key])
end
-
+
return family
end
@@ -726,7 +727,7 @@ module Hbase
descriptor.setValue(k, v)
end
end
-
+
#----------------------------------------------------------------------------------------------
# Take a snapshot of specified table
def snapshot(table, snapshot_name, *args)
@@ -783,6 +784,18 @@ module Hbase
end
#----------------------------------------------------------------------------------------------
+ # Updates the configuration of one regionserver.
+ def update_config(serverName)
+ @admin.updateConfiguration(ServerName.valueOf(serverName));
+ end
+
+ #----------------------------------------------------------------------------------------------
+ # Updates the configuration of all the regionservers.
+ def update_all_config()
+ @admin.updateConfiguration();
+ end
+
+ #----------------------------------------------------------------------------------------------
# Returns namespace's structure description
def describe_namespace(namespace_name)
namespace = @admin.getNamespaceDescriptor(namespace_name)
http://git-wip-us.apache.org/repos/asf/hbase/blob/f61df339/hbase-shell/src/main/ruby/shell.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb
index 1cfd3d6..1c10bc1 100644
--- a/hbase-shell/src/main/ruby/shell.rb
+++ b/hbase-shell/src/main/ruby/shell.rb
@@ -361,6 +361,15 @@ Shell.load_command_group(
)
Shell.load_command_group(
+ 'configuration',
+ :full_name => 'ONLINE CONFIGURATION TOOLS',
+ :commands => %w[
+ update_config
+ update_all_config
+ ]
+)
+
+Shell.load_command_group(
'quotas',
:full_name => 'CLUSTER QUOTAS TOOLS',
:commands => %w[
http://git-wip-us.apache.org/repos/asf/hbase/blob/f61df339/hbase-shell/src/main/ruby/shell/commands/update_all_config.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/update_all_config.rb b/hbase-shell/src/main/ruby/shell/commands/update_all_config.rb
new file mode 100644
index 0000000..05295b7
--- /dev/null
+++ b/hbase-shell/src/main/ruby/shell/commands/update_all_config.rb
@@ -0,0 +1,39 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+ module Commands
+ class UpdateAllConfig < Command
+ def help
+ return <<-EOF
+Reload a subset of configuration on all servers in the cluster. See
+http://hbase.apache.org/book.html?dyn_config for more details. Here is how
+you would run the command in the hbase shell:
+ hbase> update_all_config
+EOF
+ end
+
+ def command()
+ format_simple_command do
+ admin.update_all_config()
+ end
+ end
+ end
+ end
+end
http://git-wip-us.apache.org/repos/asf/hbase/blob/f61df339/hbase-shell/src/main/ruby/shell/commands/update_config.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/update_config.rb b/hbase-shell/src/main/ruby/shell/commands/update_config.rb
new file mode 100644
index 0000000..9f65fdd
--- /dev/null
+++ b/hbase-shell/src/main/ruby/shell/commands/update_config.rb
@@ -0,0 +1,40 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+ module Commands
+ class UpdateConfig < Command
+ def help
+ return <<-EOF
+Reload a subset of configuration on server 'servername' where servername is
+host, port plus startcode. For example: host187.example.com,60020,1289493121758
+See http://hbase.apache.org/book.html?dyn_config for more details. Here is how
+you would run the command in the hbase shell:
+ hbase> update_config 'servername'
+EOF
+ end
+
+ def command(serverName)
+ format_simple_command do
+ admin.update_config(serverName)
+ end
+ end
+ end
+ end
+end
http://git-wip-us.apache.org/repos/asf/hbase/blob/f61df339/src/main/docbkx/configuration.xml
----------------------------------------------------------------------
diff --git a/src/main/docbkx/configuration.xml b/src/main/docbkx/configuration.xml
index d3959b6..6907bb7 100644
--- a/src/main/docbkx/configuration.xml
+++ b/src/main/docbkx/configuration.xml
@@ -1633,5 +1633,23 @@ jconsole -J-Djavax.net.ssl.trustStore=/home/tianq/jconsoleKeyStore
</section>
<!-- important config -->
+ <section xml:id="dyn_config">
+ <title>Dynamic Configuration</title>
+ <subtitle>Changing Configuration Without Restarting Servers</subtitle>
+ <para>Since HBase 1.0.0, it is possible to change a subset of the configuration without
+ requiring a server restart. In the hbase shell, there are new operators,
+ <command>update_config</command> and <command>update_all_config</command> that
+ will prompt a server or all servers to reload configuration.</para>
+ <para>Only a subset of all configurations can currently be changed in the running server.
+ Here is an incomplete list:
+ <property>hbase.regionserver.thread.compaction.large</property>,
+ <property>hbase.regionserver.thread.compaction.small</property>,
+ <property>hbase.regionserver.thread.split</property>,
+ <property>hbase.regionserver.thread.merge</property>, as well as compaction
+ policy and configurations and adjustment to offpeak hours.
+ For the full list consult the patch attached to
+ <link xlink:href="https://issues.apache.org/jira/browse/HBASE-12147">HBASE-12147 Porting Online Config Change from 89-fb</link>.
+ </para>
+ </section>
</chapter>