You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/04/10 12:37:54 UTC
svn commit: r1311668 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/
main/java/org/apache/hadoop/hbase/regionserver/ main/ruby/ main/ruby/hbase/
test/java/org/apache/hadoop/hbase/
test/java/org/apache/hadoop/hbase/client/ test/java...
Author: mbautin
Date: Tue Apr 10 10:37:53 2012
New Revision: 1311668
URL: http://svn.apache.org/viewvc?rev=1311668&view=rev
Log:
[jira] [HBASE-5335] Dynamic Schema Config
Summary: Ability to add config options on a per-table & per-cf basis
Test Plan: - mvn test
Reviewers: kannan, liyintang, mbautin
Reviewed By: mbautin
Differential Revision: https://reviews.facebook.net/D2247
Added:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/branches/0.89-fb/src/main/ruby/hbase.rb
hbase/branches/0.89-fb/src/main/ruby/hbase/admin.rb
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java?rev=1311668&r1=1311667&r2=1311668&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java Tue Apr 10 10:37:53 2012
@@ -24,7 +24,9 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
@@ -155,9 +157,14 @@ public class HColumnDescriptor implement
*/
public static final int DEFAULT_REPLICATION_SCOPE = HConstants.REPLICATION_SCOPE_LOCAL;
- private final static Map<String, String> DEFAULT_VALUES = new HashMap<String, String>();
+ private final static Map<String, String> DEFAULT_VALUES
+ = new HashMap<String, String>();
+ private final static Set<ImmutableBytesWritable> RESERVED_KEYWORDS
+ = new HashSet<ImmutableBytesWritable>();
static {
DEFAULT_VALUES.put(BLOOMFILTER, DEFAULT_BLOOMFILTER);
+ DEFAULT_VALUES.put(BLOOMFILTER_ERRORRATE,
+ String.valueOf(DEFAULT_BLOOMFILTER_ERROR_RATE));
DEFAULT_VALUES.put(REPLICATION_SCOPE, String.valueOf(DEFAULT_REPLICATION_SCOPE));
DEFAULT_VALUES.put(HConstants.VERSIONS, String.valueOf(DEFAULT_VERSIONS));
DEFAULT_VALUES.put(COMPRESSION, DEFAULT_COMPRESSION);
@@ -169,14 +176,16 @@ public class HColumnDescriptor implement
String.valueOf(DEFAULT_ENCODE_ON_DISK));
DEFAULT_VALUES.put(DATA_BLOCK_ENCODING,
String.valueOf(DEFAULT_DATA_BLOCK_ENCODING));
+ for (String s : DEFAULT_VALUES.keySet()) {
+ RESERVED_KEYWORDS.add(new ImmutableBytesWritable(Bytes.toBytes(s)));
+ }
}
-
// Column family name
private byte [] name;
// Column metadata
- protected Map<ImmutableBytesWritable,ImmutableBytesWritable> values =
+ protected final Map<ImmutableBytesWritable, ImmutableBytesWritable> values =
new HashMap<ImmutableBytesWritable,ImmutableBytesWritable>();
/*
@@ -415,6 +424,7 @@ public class HColumnDescriptor implement
* @return All values.
*/
public Map<ImmutableBytesWritable,ImmutableBytesWritable> getValues() {
+ // shallow pointer copy
return Collections.unmodifiableMap(values);
}
@@ -442,7 +452,11 @@ public class HColumnDescriptor implement
* @return this (for chained invocation)
*/
public HColumnDescriptor setValue(String key, String value) {
- setValue(Bytes.toBytes(key), Bytes.toBytes(value));
+ if (value == null) {
+ remove(Bytes.toBytes(key));
+ } else {
+ setValue(Bytes.toBytes(key), Bytes.toBytes(value));
+ }
return this;
}
@@ -700,16 +714,7 @@ public class HColumnDescriptor implement
s.append(" => '");
s.append(Bytes.toString(name));
s.append("'");
- for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
- values.entrySet()) {
- String key = Bytes.toString(e.getKey().get());
- String value = Bytes.toString(e.getValue().get());
- s.append(", ");
- s.append(key);
- s.append(" => '");
- s.append(value);
- s.append("'");
- }
+ s.append(getValues(true));
s.append('}');
return s.toString();
}
@@ -722,36 +727,61 @@ public class HColumnDescriptor implement
s.append(" => '");
s.append(Bytes.toString(name));
s.append("'");
- for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
- values.entrySet()) {
- String key = Bytes.toString(e.getKey().get());
- String value = Bytes.toString(e.getValue().get());
- if(defaults.get(key) == null || !defaults.get(key).equalsIgnoreCase(value)) {
- s.append(", ");
- s.append(key);
- s.append(" => '");
- s.append(value);
- s.append("'");
- }
- }
+ s.append(getValues(false));
s.append('}');
return s.toString();
}
+ private StringBuilder getValues(boolean printDefaults) {
+ StringBuilder s = new StringBuilder();
+
+ boolean hasAdvancedKeys = false;
+
+ // print all reserved keys first
+ for (ImmutableBytesWritable k : values.keySet()) {
+ if (!RESERVED_KEYWORDS.contains(k)) {
+ hasAdvancedKeys = true;
+ continue;
+ }
+ String key = Bytes.toString(k.get());
+ String value = Bytes.toString(values.get(k).get());
+ if (printDefaults
+ || !DEFAULT_VALUES.containsKey(key)
+ || !DEFAULT_VALUES.get(key).equalsIgnoreCase(value)) {
+ s.append(", ");
+ s.append(key);
+ s.append(" => ");
+ s.append('\'').append(value).append('\'');
+ }
+ }
- public static Map<String, String>getDefaultValues() {
- Map<String, String> defaultValues = new HashMap<String, String>();
+ // print all other keys as advanced options
+ if (hasAdvancedKeys) {
+ s.append(", ");
+ s.append("{").append(HConstants.CONFIG).append(" => ");
+ s.append('{');
+ boolean printComma = false;
+ for (ImmutableBytesWritable k : values.keySet()) {
+ if (RESERVED_KEYWORDS.contains(k)) {
+ continue;
+ }
+ String key = Bytes.toString(k.get());
+ String value = Bytes.toString(values.get(k).get());
+ if (printComma) {
+ s.append(", ");
+ }
+ printComma = true;
+ s.append('\'').append(key).append('\'');
+ s.append(" => ");
+ s.append('\'').append(value).append('\'');
+ }
+ s.append('}');
+ }
+ return s;
+ }
- defaultValues.put(BLOOMFILTER, DEFAULT_BLOOMFILTER);
- defaultValues.put(REPLICATION_SCOPE, String.valueOf(DEFAULT_REPLICATION_SCOPE));
- defaultValues.put(HConstants.VERSIONS, String.valueOf(DEFAULT_VERSIONS));
- defaultValues.put(COMPRESSION, DEFAULT_COMPRESSION);
- defaultValues.put(TTL, String.valueOf(DEFAULT_TTL));
- defaultValues.put(BLOCKSIZE, String.valueOf(DEFAULT_BLOCKSIZE));
- defaultValues.put(HConstants.IN_MEMORY, String.valueOf(DEFAULT_IN_MEMORY));
- defaultValues.put(BLOOMFILTER_ERRORRATE, String.valueOf(DEFAULT_BLOOMFILTER_ERROR_RATE));
- defaultValues.put(BLOCKCACHE, String.valueOf(DEFAULT_BLOCKCACHE));
- return defaultValues;
+ public static Map<String, String> getDefaultValues() {
+ return Collections.unmodifiableMap(DEFAULT_VALUES);
}
/**
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1311668&r1=1311667&r2=1311668&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java Tue Apr 10 10:37:53 2012
@@ -364,6 +364,7 @@ public final class HConstants {
public static final String NAME = "NAME";
public static final String VERSIONS = "VERSIONS";
public static final String IN_MEMORY = "IN_MEMORY";
+ public static final String CONFIG = "CONFIG";
/**
* This is a retry backoff multiplier table similar to the BSD TCP syn
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java?rev=1311668&r1=1311667&r2=1311668&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java Tue Apr 10 10:37:53 2012
@@ -25,10 +25,12 @@ import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+import java.util.TreeSet;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -53,7 +55,7 @@ public class HTableDescriptor implements
private String nameAsString = "";
// Table metadata
- protected Map<ImmutableBytesWritable, ImmutableBytesWritable> values =
+ protected final Map<ImmutableBytesWritable, ImmutableBytesWritable> values =
new HashMap<ImmutableBytesWritable, ImmutableBytesWritable>();
public static final String FAMILIES = "FAMILIES";
@@ -97,6 +99,24 @@ public class HTableDescriptor implements
public static final boolean DEFAULT_DEFERRED_LOG_FLUSH = true;
+ private final static Map<String, String> DEFAULT_VALUES
+ = new HashMap<String, String>();
+ private final static Set<ImmutableBytesWritable> RESERVED_KEYWORDS
+ = new HashSet<ImmutableBytesWritable>();
+ static {
+ DEFAULT_VALUES.put(MAX_FILESIZE, String.valueOf(DEFAULT_MAX_FILESIZE));
+ DEFAULT_VALUES.put(READONLY, String.valueOf(DEFAULT_READONLY));
+ DEFAULT_VALUES.put(MEMSTORE_FLUSHSIZE,
+ String.valueOf(DEFAULT_MEMSTORE_FLUSH_SIZE));
+ DEFAULT_VALUES.put(DEFERRED_LOG_FLUSH,
+ String.valueOf(DEFAULT_DEFERRED_LOG_FLUSH));
+ for (String s : DEFAULT_VALUES.keySet()) {
+ RESERVED_KEYWORDS.add(new ImmutableBytesWritable(Bytes.toBytes(s)));
+ }
+ RESERVED_KEYWORDS.add(IS_ROOT_KEY);
+ RESERVED_KEYWORDS.add(IS_META_KEY);
+ }
+
private volatile Boolean meta = null;
private volatile Boolean root = null;
private Boolean isDeferredLog = null;
@@ -314,7 +334,8 @@ public class HTableDescriptor implements
* @return All values.
*/
public Map<ImmutableBytesWritable,ImmutableBytesWritable> getValues() {
- return Collections.unmodifiableMap(values);
+ // shallow pointer copy
+ return Collections.unmodifiableMap(values);
}
/**
@@ -348,7 +369,11 @@ public class HTableDescriptor implements
* @param value The value.
*/
public void setValue(String key, String value) {
- setValue(Bytes.toBytes(key), Bytes.toBytes(value));
+ if (value == null) {
+ remove(Bytes.toBytes(key));
+ } else {
+ setValue(Bytes.toBytes(key), Bytes.toBytes(value));
+ }
}
/**
@@ -469,79 +494,89 @@ public class HTableDescriptor implements
@Override
public String toString() {
StringBuilder s = new StringBuilder();
- s.append('{');
- s.append(HConstants.NAME);
- s.append(" => '");
- s.append(Bytes.toString(name));
- s.append("'");
- for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
- values.entrySet()) {
- String key = Bytes.toString(e.getKey().get());
- String value = Bytes.toString(e.getValue().get());
- if (key == null) {
- continue;
- }
- String upperCase = key.toUpperCase();
- if (upperCase.equals(IS_ROOT) || upperCase.equals(IS_META)) {
- // Skip. Don't bother printing out read-only values if false.
- if (value.toLowerCase().equals(Boolean.FALSE.toString())) {
- continue;
- }
- }
- s.append(", ");
- s.append(Bytes.toString(e.getKey().get()));
- s.append(" => '");
- s.append(Bytes.toString(e.getValue().get()));
- s.append("'");
- }
- s.append(", ");
- s.append(FAMILIES);
- s.append(" => ");
- s.append(families.values());
- s.append('}');
+ s.append('\'').append(Bytes.toString(name)).append('\'');
+ s.append(getValues(true));
+ for (HColumnDescriptor f : families.values()) {
+ s.append(", ").append(f);
+ }
return s.toString();
}
public String toStringCustomizedValues() {
StringBuilder s = new StringBuilder();
- s.append('{');
- s.append(HConstants.NAME);
- s.append(" => '");
- s.append(Bytes.toString(name));
- s.append("'");
- for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
- values.entrySet()) {
- String key = Bytes.toString(e.getKey().get());
- String value = Bytes.toString(e.getValue().get());
- if (key == null) {
+ s.append('\'').append(Bytes.toString(name)).append('\'');
+ s.append(getValues(false));
+ for(HColumnDescriptor hcd : families.values()) {
+ s.append(", ").append(hcd.toStringCustomizedValues());
+ }
+ return s.toString();
+ }
+
+ private StringBuilder getValues(boolean printDefaults) {
+ StringBuilder s = new StringBuilder();
+
+ // step 1: set partitioning and pruning
+ Set<ImmutableBytesWritable> reservedKeys = new TreeSet<ImmutableBytesWritable>();
+ Set<ImmutableBytesWritable> configKeys = new TreeSet<ImmutableBytesWritable>();
+ for (ImmutableBytesWritable k : values.keySet()) {
+ if (!RESERVED_KEYWORDS.contains(k)) {
+ configKeys.add(k);
continue;
}
- String upperCase = key.toUpperCase();
- if (upperCase.equals(IS_ROOT) || upperCase.equals(IS_META)) {
- // Skip. Don't bother printing out read-only values if false.
- if (value.toLowerCase().equals(Boolean.FALSE.toString())) {
- continue;
- }
+ // only print out IS_ROOT/IS_META if true
+ String key = Bytes.toString(k.get());
+ String value = Bytes.toString(values.get(k).get());
+ if (key.equalsIgnoreCase(IS_ROOT) || key.equalsIgnoreCase(IS_META)) {
+ if (Boolean.valueOf(value) == false) continue;
}
+ if (printDefaults
+ || !DEFAULT_VALUES.containsKey(key)
+ || !DEFAULT_VALUES.get(key).equalsIgnoreCase(value)) {
+ reservedKeys.add(k);
+ }
+ }
+
+ // early exit optimization
+ if (reservedKeys.isEmpty() && configKeys.isEmpty()) return s;
+
+ // step 2: printing
+ s.append(", {METHOD => 'table_att'");
+
+ // print all reserved keys first
+ for (ImmutableBytesWritable k : reservedKeys) {
+ String key = Bytes.toString(k.get());
+ String value = Bytes.toString(values.get(k).get());
s.append(", ");
- s.append(Bytes.toString(e.getKey().get()));
- s.append(" => '");
- s.append(Bytes.toString(e.getValue().get()));
- s.append("'");
- }
- s.append(", ");
- s.append(FAMILIES);
- s.append(" => [");
- int size = families.values().size();
- int i = 0;
- for(HColumnDescriptor hcd : families.values()) {
- s.append(hcd.toStringCustomizedValues());
- i++;
- if( i != size)
- s.append(", ");
+ s.append(key);
+ s.append(" => ");
+ s.append('\'').append(value).append('\'');
}
- s.append("]}");
- return s.toString();
+
+ if (!configKeys.isEmpty()) {
+ // print all non-reserved, advanced config keys as a separate subset
+ s.append(", ");
+ s.append(HConstants.CONFIG).append(" => ");
+ s.append("{");
+ boolean printComma = false;
+ for (ImmutableBytesWritable k : configKeys) {
+ String key = Bytes.toString(k.get());
+ String value = Bytes.toString(values.get(k).get());
+ if (printComma) s.append(", ");
+ printComma = true;
+ s.append('\'').append(key).append('\'');
+ s.append(" => ");
+ s.append('\'').append(value).append('\'');
+ }
+ s.append('}');
+ }
+
+ s.append('}'); // end METHOD
+
+ return s;
+ }
+
+ public static Map<String, String> getDefaultValues() {
+ return Collections.unmodifiableMap(DEFAULT_VALUES);
}
/**
Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java?rev=1311668&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java (added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompoundConfiguration.java Tue Apr 10 10:37:53 2012
@@ -0,0 +1,460 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Do a shallow merge of multiple KV configuration pools. This is a very useful
+ * utility class to easily add per-object configurations in addition to wider
+ * scope settings. This is different from Configuration.addResource()
+ * functionality, which performs a deep merge and mutates the common data
+ * structure.
+ * <p>
+ * For clarity: the shallow merge allows the user to mutate either of the
+ * configuration objects and have changes reflected everywhere. In contrast to a
+ * deep merge, that requires you to explicitly know all applicable copies to
+ * propagate changes.
+ */
+class CompoundConfiguration extends Configuration {
+ /**
+ * Default Constructor. Initializes empty configuration
+ */
+ public CompoundConfiguration() {
+ }
+
+ // Devs: these APIs are the same contract as their counterparts in
+ // Configuration.java
+ private static interface ImmutableConfigMap {
+ String get(String key);
+ String getRaw(String key);
+ Class<?> getClassByName(String name) throws ClassNotFoundException;
+ int size();
+ }
+
+ protected List<ImmutableConfigMap> configs
+ = new ArrayList<ImmutableConfigMap>();
+
+ /****************************************************************************
+ * These initial APIs actually required original thought
+ ***************************************************************************/
+
+ /**
+ * Add Hadoop Configuration object to config list
+ * @param conf configuration object
+ * @return this, for builder pattern
+ */
+ public CompoundConfiguration add(final Configuration conf) {
+ if (conf instanceof CompoundConfiguration) {
+ this.configs.addAll(0, ((CompoundConfiguration) conf).configs);
+ return this;
+ }
+ // put new config at the front of the list (top priority)
+ this.configs.add(0, new ImmutableConfigMap() {
+ Configuration c = conf;
+
+ @Override
+ public String get(String key) {
+ return c.get(key);
+ }
+
+ @Override
+ public String getRaw(String key) {
+ return c.getRaw(key);
+ }
+
+ @Override
+ public Class<?> getClassByName(String name)
+ throws ClassNotFoundException {
+ return c.getClassByName(name);
+ }
+
+ @Override
+ public int size() {
+ return c.size();
+ }
+
+ @Override
+ public String toString() {
+ return c.toString();
+ }
+ });
+ return this;
+ }
+
+ /**
+ * Add ImmutableBytesWritable map to config list. This map is generally
+ * created by HTableDescriptor or HColumnDescriptor, but can be abstractly
+ * used.
+ *
+ * @param map
+ * ImmutableBytesWritable map
+ * @return this, for builder pattern
+ */
+ public CompoundConfiguration add(
+ final Map<ImmutableBytesWritable, ImmutableBytesWritable> map) {
+ // put new map at the front of the list (top priority)
+ this.configs.add(0, new ImmutableConfigMap() {
+ Map<ImmutableBytesWritable, ImmutableBytesWritable> m = map;
+
+ @Override
+ public String get(String key) {
+ ImmutableBytesWritable ibw = new ImmutableBytesWritable(Bytes
+ .toBytes(key));
+ if (!m.containsKey(ibw))
+ return null;
+ ImmutableBytesWritable value = m.get(ibw);
+ if (value == null || value.get() == null)
+ return null;
+ return Bytes.toString(value.get());
+ }
+
+ @Override
+ public String getRaw(String key) {
+ return get(key);
+ }
+
+ @Override
+ public Class<?> getClassByName(String name)
+ throws ClassNotFoundException {
+ return null;
+ }
+
+ @Override
+ public int size() {
+ // TODO Auto-generated method stub
+ return m.size();
+ }
+
+ @Override
+ public String toString() {
+ return m.toString();
+ }
+ });
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append("CompoundConfiguration: " + this.configs.size() + " configs");
+ for (ImmutableConfigMap m : this.configs) {
+ sb.append(this.configs);
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public String get(String key) {
+ for (ImmutableConfigMap m : this.configs) {
+ String value = m.get(key);
+ if (value != null) {
+ return value;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public String getRaw(String key) {
+ for (ImmutableConfigMap m : this.configs) {
+ String value = m.getRaw(key);
+ if (value != null) {
+ return value;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Class<?> getClassByName(String name) throws ClassNotFoundException {
+ for (ImmutableConfigMap m : this.configs) {
+ try {
+ Class<?> value = m.getClassByName(name);
+ if (value != null) {
+ return value;
+ }
+ } catch (ClassNotFoundException e) {
+ // don't propagate an exception until all configs fail
+ continue;
+ }
+ }
+ throw new ClassNotFoundException();
+ }
+
+ @Override
+ public int size() {
+ int ret = 0;
+ for (ImmutableConfigMap m : this.configs) {
+ ret += m.size();
+ }
+ return ret;
+ }
+
+ /***************************************************************************
+ * You should just ignore everything below this line unless there's a bug in
+ * Configuration.java...
+ *
+ * Below get APIs are directly copied from Configuration.java Oh, how I wish
+ * this wasn't so! A tragically-sad example of why you use interfaces instead
+ * of inheritance.
+ *
+ * Why the duplication? We basically need to override Configuration.getProps
+ * or we'd need protected access to Configuration.properties so we can modify
+ * that pointer. There are a bunch of functions in the base Configuration that
+ * call getProps() and we need to use our derived version instead of the base
+ * version. We need to make a generic implementation that works across all
+ * HDFS versions. We should modify Configuration.properties in HDFS 1.0 to be
+ * protected, but we still need to have this code until that patch makes it to
+ * all the HDFS versions we support.
+ ***************************************************************************/
+
+ @Override
+ public String get(String name, String defaultValue) {
+ String ret = get(name);
+ return ret == null ? defaultValue : ret;
+ }
+
+ @Override
+ public int getInt(String name, int defaultValue) {
+ String valueString = get(name);
+ if (valueString == null)
+ return defaultValue;
+ try {
+ String hexString = getHexDigits(valueString);
+ if (hexString != null) {
+ return Integer.parseInt(hexString, 16);
+ }
+ return Integer.parseInt(valueString);
+ } catch (NumberFormatException e) {
+ return defaultValue;
+ }
+ }
+
+ @Override
+ public long getLong(String name, long defaultValue) {
+ String valueString = get(name);
+ if (valueString == null)
+ return defaultValue;
+ try {
+ String hexString = getHexDigits(valueString);
+ if (hexString != null) {
+ return Long.parseLong(hexString, 16);
+ }
+ return Long.parseLong(valueString);
+ } catch (NumberFormatException e) {
+ return defaultValue;
+ }
+ }
+
+ protected String getHexDigits(String value) {
+ boolean negative = false;
+ String str = value;
+ String hexString = null;
+ if (value.startsWith("-")) {
+ negative = true;
+ str = value.substring(1);
+ }
+ if (str.startsWith("0x") || str.startsWith("0X")) {
+ hexString = str.substring(2);
+ if (negative) {
+ hexString = "-" + hexString;
+ }
+ return hexString;
+ }
+ return null;
+ }
+
+ @Override
+ public float getFloat(String name, float defaultValue) {
+ String valueString = get(name);
+ if (valueString == null)
+ return defaultValue;
+ try {
+ return Float.parseFloat(valueString);
+ } catch (NumberFormatException e) {
+ return defaultValue;
+ }
+ }
+
+ @Override
+ public boolean getBoolean(String name, boolean defaultValue) {
+ String valueString = get(name);
+ if ("true".equals(valueString))
+ return true;
+ else if ("false".equals(valueString))
+ return false;
+ else return defaultValue;
+ }
+
+ @Override
+ public IntegerRanges getRange(String name, String defaultValue) {
+ return new IntegerRanges(get(name, defaultValue));
+ }
+
+ @Override
+ public Collection<String> getStringCollection(String name) {
+ String valueString = get(name);
+ return StringUtils.getStringCollection(valueString);
+ }
+
+ @Override
+ public String[] getStrings(String name) {
+ String valueString = get(name);
+ return StringUtils.getStrings(valueString);
+ }
+
+ @Override
+ public String[] getStrings(String name, String... defaultValue) {
+ String valueString = get(name);
+ if (valueString == null) {
+ return defaultValue;
+ } else {
+ return StringUtils.getStrings(valueString);
+ }
+ }
+
+ @Override
+ public Class<?>[] getClasses(String name, Class<?>... defaultValue) {
+ String[] classnames = getStrings(name);
+ if (classnames == null)
+ return defaultValue;
+ try {
+ Class<?>[] classes = new Class<?>[classnames.length];
+ for (int i = 0; i < classnames.length; i++) {
+ classes[i] = getClassByName(classnames[i]);
+ }
+ return classes;
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Class<?> getClass(String name, Class<?> defaultValue) {
+ String valueString = get(name);
+ if (valueString == null)
+ return defaultValue;
+ try {
+ return getClassByName(valueString);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public <U> Class<? extends U> getClass(String name,
+ Class<? extends U> defaultValue, Class<U> xface) {
+ try {
+ Class<?> theClass = getClass(name, defaultValue);
+ if (theClass != null && !xface.isAssignableFrom(theClass))
+ throw new RuntimeException(theClass + " not " + xface.getName());
+ else if (theClass != null)
+ return theClass.asSubclass(xface);
+ else
+ return null;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /*******************************************************************
+ * This class is immutable. Quickly abort any attempts to alter it *
+ *******************************************************************/
+
+ @Override
+ public void clear() {
+ throw new UnsupportedOperationException("Immutable Configuration");
+ }
+
+ @Override
+ public Iterator<Map.Entry<String, String>> iterator() {
+ throw new UnsupportedOperationException("Immutable Configuration");
+ }
+
+ @Override
+ public void set(String name, String value) {
+ throw new UnsupportedOperationException("Immutable Configuration");
+ }
+ @Override
+ public void setIfUnset(String name, String value) {
+ throw new UnsupportedOperationException("Immutable Configuration");
+ }
+ @Override
+ public void setInt(String name, int value) {
+ throw new UnsupportedOperationException("Immutable Configuration");
+ }
+ @Override
+ public void setLong(String name, long value) {
+ throw new UnsupportedOperationException("Immutable Configuration");
+ }
+ @Override
+ public void setFloat(String name, float value) {
+ throw new UnsupportedOperationException("Immutable Configuration");
+ }
+ @Override
+ public void setBoolean(String name, boolean value) {
+ throw new UnsupportedOperationException("Immutable Configuration");
+ }
+ @Override
+ public void setBooleanIfUnset(String name, boolean value) {
+ throw new UnsupportedOperationException("Immutable Configuration");
+ }
+ @Override
+ public void setStrings(String name, String... values) {
+ throw new UnsupportedOperationException("Immutable Configuration");
+ }
+ @Override
+ public void setClass(String name, Class<?> theClass, Class<?> xface) {
+ throw new UnsupportedOperationException("Immutable Configuration");
+ }
+ @Override
+ public void setClassLoader(ClassLoader classLoader) {
+ throw new UnsupportedOperationException("Immutable Configuration");
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ throw new UnsupportedOperationException("Immutable Configuration");
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ throw new UnsupportedOperationException("Immutable Configuration");
+ }
+
+ @Override
+ public void writeXml(OutputStream out) throws IOException {
+ throw new UnsupportedOperationException("Immutable Configuration");
+ }
+};
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1311668&r1=1311667&r2=1311668&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue Apr 10 10:37:53 2012
@@ -192,6 +192,7 @@ public class HRegion implements HeapSize
final HLog log;
final FileSystem fs;
final Configuration conf;
+ final Configuration baseConf;
final HRegionInfo regionInfo;
final Path regiondir;
KeyValue.KVComparator comparator;
@@ -401,6 +402,7 @@ public class HRegion implements HeapSize
this.tableDir = null;
this.blockingMemStoreSize = 0L;
this.conf = null;
+ this.baseConf = null;
this.flushListener = null;
this.fs = null;
this.timestampTooNew = HConstants.LATEST_TIMESTAMP;
@@ -413,6 +415,16 @@ public class HRegion implements HeapSize
}
/**
+ * HRegion copy constructor. Useful when reopening a closed region (normally
+ * for unit tests)
+ * @param other original object
+ */
+ public HRegion(HRegion other) {
+ this(other.getTableDir(), other.getLog(), other.getFilesystem(),
+ other.baseConf, other.getRegionInfo(), null);
+ }
+
+ /**
* HRegion constructor. his constructor should only be used for testing and
* extensions. Instances of HRegion should be instantiated with the
* {@link HRegion#newHRegion(Path, HLog, FileSystem, Configuration, org.apache.hadoop.hbase.HRegionInfo, FlushRequester)} method.
@@ -438,13 +450,21 @@ public class HRegion implements HeapSize
* @see HRegion#newHRegion(Path, HLog, FileSystem, Configuration, org.apache.hadoop.hbase.HRegionInfo, FlushRequester)
*/
- public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
- final HRegionInfo regionInfo, FlushRequester flushListener) {
+ public HRegion(Path tableDir, HLog log, FileSystem fs,
+ Configuration confParam, final HRegionInfo regionInfo,
+ FlushRequester flushListener) {
this.tableDir = tableDir;
this.comparator = regionInfo.getComparator();
this.log = log;
this.fs = fs;
- this.conf = conf;
+ if (confParam instanceof CompoundConfiguration) {
+ throw new IllegalArgumentException("Need original base configuration");
+ }
+ // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor
+ this.baseConf = confParam;
+ this.conf = new CompoundConfiguration()
+ .add(confParam)
+ .add(regionInfo.getTableDesc().getValues());
this.regionInfo = regionInfo;
this.flushListener = flushListener;
this.threadWakeFrequency = conf.getLong(HConstants.THREAD_WAKE_FREQUENCY,
@@ -928,11 +948,6 @@ public class HRegion implements HeapSize
return this.log;
}
- /** @return Configuration object */
- public Configuration getConf() {
- return this.conf;
- }
-
/** @return region directory Path */
public Path getRegionDir() {
return this.regiondir;
@@ -1051,10 +1066,10 @@ public class HRegion implements HeapSize
// Create a region instance and then move the splits into place under
// regionA and regionB.
HRegion regionA =
- HRegion.newHRegion(tableDir, log, fs, conf, regionAInfo, null);
+ HRegion.newHRegion(tableDir, log, fs, baseConf, regionAInfo, null);
moveInitialFilesIntoPlace(this.fs, dirA, regionA.getRegionDir());
HRegion regionB =
- HRegion.newHRegion(tableDir, log, fs, conf, regionBInfo, null);
+ HRegion.newHRegion(tableDir, log, fs, baseConf, regionBInfo, null);
moveInitialFilesIntoPlace(this.fs, dirB, regionB.getRegionDir());
return new HRegion [] {regionA, regionB};
@@ -3421,7 +3436,7 @@ public class HRegion implements HeapSize
listPaths(fs, b.getRegionDir());
}
- Configuration conf = a.getConf();
+ Configuration conf = a.baseConf;
HTableDescriptor tabledesc = a.getTableDesc();
HLog log = a.getLog();
Path tableDir = a.getTableDir();
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java?rev=1311668&r1=1311667&r2=1311668&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java Tue Apr 10 10:37:53 2012
@@ -38,7 +38,7 @@ class SplitRequest implements Runnable {
public void run() {
HRegionServer server = region.getRegionServer();
try {
- Configuration conf = region.getConf();
+ Configuration conf = server.getConfiguration();
final HRegionInfo oldRegionInfo = region.getRegionInfo();
final long startTime = System.currentTimeMillis();
final HRegion[] newRegions = region.splitRegion(midKey);
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1311668&r1=1311667&r2=1311668&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Tue Apr 10 10:37:53 2012
@@ -164,9 +164,10 @@ public class Store extends SchemaConfigu
* @throws IOException
*/
protected Store(Path basedir, HRegion region, HColumnDescriptor family,
- FileSystem fs, Configuration conf)
+ FileSystem fs, Configuration confParam)
throws IOException {
- super(conf, region.getTableDesc().getNameAsString(),
+ super(new CompoundConfiguration().add(confParam).add(
+ family.getValues()), region.getTableDesc().getNameAsString(),
Bytes.toString(family.getName()));
HRegionInfo info = region.regionInfo;
this.fs = fs;
@@ -178,7 +179,10 @@ public class Store extends SchemaConfigu
}
this.region = region;
this.family = family;
- this.conf = conf;
+ // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor
+ this.conf = new CompoundConfiguration()
+ .add(confParam)
+ .add(family.getValues());
this.blockcache = family.isBlockCacheEnabled();
this.blocksize = family.getBlocksize();
this.compression = family.getCompression();
Modified: hbase/branches/0.89-fb/src/main/ruby/hbase.rb
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/ruby/hbase.rb?rev=1311668&r1=1311667&r2=1311668&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/ruby/hbase.rb (original)
+++ hbase/branches/0.89-fb/src/main/ruby/hbase.rb Tue Apr 10 10:37:53 2012
@@ -44,6 +44,7 @@ module HBaseConstants
NAME = HConstants::NAME
VERSIONS = HConstants::VERSIONS
IN_MEMORY = HConstants::IN_MEMORY
+ CONFIG = HConstants::CONFIG
STOPROW = "STOPROW"
STARTROW = "STARTROW"
ENDROW = STOPROW
Modified: hbase/branches/0.89-fb/src/main/ruby/hbase/admin.rb
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/ruby/hbase/admin.rb?rev=1311668&r1=1311667&r2=1311668&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/ruby/hbase/admin.rb (original)
+++ hbase/branches/0.89-fb/src/main/ruby/hbase/admin.rb Tue Apr 10 10:37:53 2012
@@ -159,16 +159,43 @@ module Hbase
raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type")
end
- if arg.kind_of?(Hash) and (arg.has_key?(NUMREGIONS) or arg.has_key?(SPLITALGO))
- raise(ArgumentError, "Column family configuration should be specified in a separate clause") if arg.has_key?(NAME)
- raise(ArgumentError, "Number of regions must be specified") unless arg.has_key?(NUMREGIONS)
- raise(ArgumentError, "Split algorithm must be specified") unless arg.has_key?(SPLITALGO)
- raise(ArgumentError, "Number of regions must be greater than 1") unless arg[NUMREGIONS] > 1
- num_regions = arg[NUMREGIONS]
- split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg[SPLITALGO])
- else
- # Add column to the table
+ if arg.kind_of?(String)
+ # the arg is a string, default action is to add a column to the table
htd.addFamily(hcd(arg, htd))
+ else
+ # arg is a hash. 3 possibilities:
+ if (arg.has_key?(NUMREGIONS) or arg.has_key?(SPLITALGO))
+ # (1) deprecated region pre-split API
+ raise(ArgumentError, "Column family configuration should be specified in a separate clause") if arg.has_key?(NAME)
+ raise(ArgumentError, "Number of regions must be specified") unless arg.has_key?(NUMREGIONS)
+ raise(ArgumentError, "Split algorithm must be specified") unless arg.has_key?(SPLITALGO)
+ raise(ArgumentError, "Number of regions must be greater than 1") unless arg[NUMREGIONS] > 1
+ num_regions = arg[NUMREGIONS]
+ split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg[SPLITALGO])
+ elsif (method = arg.delete(METHOD))
+ # (2) table_attr modification
+ raise(ArgumentError, "table_att is currently the only supported method") unless method == 'table_att'
+ raise(ArgumentError, "NUMREGIONS & SPLITALGO must both be specified") unless arg.has_key?(NUMREGIONS) == arg.has_key?(split_algo)
+ htd.setMaxFileSize(JLong.valueOf(arg[MAX_FILESIZE])) if arg[MAX_FILESIZE]
+ htd.setReadOnly(JBoolean.valueOf(arg[READONLY])) if arg[READONLY]
+ htd.setMemStoreFlushSize(JLong.valueOf(arg[MEMSTORE_FLUSHSIZE])) if arg[MEMSTORE_FLUSHSIZE]
+ htd.setDeferredLogFlush(JBoolean.valueOf(arg[DEFERRED_LOG_FLUSH])) if arg[DEFERRED_LOG_FLUSH]
+ if arg[NUMREGIONS]
+ raise(ArgumentError, "Number of regions must be greater than 1") unless arg[NUMREGIONS] > 1
+ num_regions = arg[NUMREGIONS]
+ split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg[SPLITALGO])
+ end
+ if arg[CONFIG]
+ raise(ArgumentError, "#{CONFIG} must be a Hash type") unless arg.kind_of?(Hash)
+ for k,v in arg[CONFIG]
+ v = v.to_s unless v.nil?
+ htd.setValue(k, v)
+ end
+ end
+ else
+ # (3) column family spec
+ htd.addFamily(hcd(arg, htd))
+ end
end
end
@@ -317,6 +344,13 @@ module Hbase
htd.setReadOnly(JBoolean.valueOf(arg[READONLY])) if arg[READONLY]
htd.setMemStoreFlushSize(JLong.valueOf(arg[MEMSTORE_FLUSHSIZE])) if arg[MEMSTORE_FLUSHSIZE]
htd.setDeferredLogFlush(JBoolean.valueOf(arg[DEFERRED_LOG_FLUSH])) if arg[DEFERRED_LOG_FLUSH]
+ if arg[CONFIG]
+ raise(ArgumentError, "#{CONFIG} must be a Hash type") unless arg.kind_of?(Hash)
+ for k,v in arg[CONFIG]
+ v = v.to_s unless v.nil?
+ htd.setValue(k, v)
+ end
+ end
@admin.modifyTable(table_name.to_java_bytes, htd)
next
end
@@ -417,6 +451,14 @@ module Hbase
family.setEncodeOnDisk(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::ENCODE_ON_DISK])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::ENCODE_ON_DISK)
family.setBlocksize(JInteger.valueOf(arg[HColumnDescriptor::BLOCKSIZE])) if arg.include?(HColumnDescriptor::BLOCKSIZE)
family.setMaxVersions(JInteger.valueOf(arg[VERSIONS])) if arg.include?(HColumnDescriptor::VERSIONS)
+ if arg[CONFIG]
+ raise(ArgumentError, "#{CONFIG} must be a Hash type") unless arg.kind_of?(Hash)
+ for k,v in arg[CONFIG]
+ v = v.to_s unless v.nil?
+ end
+ family.setValue(k, v)
+ end
+ end
return family
end
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java?rev=1311668&r1=1311667&r2=1311668&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java Tue Apr 10 10:37:53 2012
@@ -158,9 +158,7 @@ public abstract class HBaseTestCase exte
protected HRegion openClosedRegion(final HRegion closedRegion)
throws IOException {
- HRegion r = new HRegion(closedRegion.getTableDir(), closedRegion.getLog(),
- closedRegion.getFilesystem(), closedRegion.getConf(),
- closedRegion.getRegionInfo(), null);
+ HRegion r = new HRegion(closedRegion);
r.initialize();
return r;
}
Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java?rev=1311668&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java Tue Apr 10 10:37:53 2012
@@ -0,0 +1,242 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class TestFromClientSide3 {
+ final Log LOG = LogFactory.getLog(getClass());
+ private final static HBaseTestingUtility TEST_UTIL
+ = new HBaseTestingUtility();
+ private static byte[] ROW = Bytes.toBytes("testRow");
+ private static byte[] FAMILY = Bytes.toBytes("testFamily");
+ private static byte[] QUALIFIER = Bytes.toBytes("testQualifier");
+ private static byte[] VALUE = Bytes.toBytes("testValue");
+ private static Random random = new Random();
+ private static int SLAVES = 3;
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ TEST_UTIL.startMiniCluster(SLAVES);
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @Before
+ public void setUp() throws Exception {
+ // Nothing to do.
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @After
+ public void tearDown() throws Exception {
+ // Nothing to do.
+ }
+
+ private void randomCFPuts(HTable table, byte[] row, byte[] family, int nPuts)
+ throws Exception {
+ Put put = new Put(row);
+ for (int i = 0; i < nPuts; i++) {
+ byte[] qualifier = Bytes.toBytes(random.nextInt());
+ byte[] value = Bytes.toBytes(random.nextInt());
+ put.add(family, qualifier, value);
+ }
+ table.put(put);
+ }
+
+ private void performMultiplePutAndFlush(HBaseAdmin admin, HTable table,
+ byte[] row, byte[] family, int nFlushes, int nPuts) throws Exception {
+
+ // connection needed for poll-wait
+ HConnection conn = HConnectionManager.getConnection(TEST_UTIL
+ .getConfiguration());
+ HRegionLocation loc = table.getRegionLocation(row);
+ HRegionInterface server = conn.getHRegionConnection(loc.getServerAddress());
+ byte[] regName = loc.getRegionInfo().getRegionName();
+
+ for (int i = 0; i < nFlushes; i++) {
+ randomCFPuts(table, row, family, nPuts);
+ int sfCount = server.getStoreFileList(regName, FAMILY).size();
+
+ // TODO: replace this api with a synchronous flush after HBASE-2949
+ admin.flush(table.getTableName());
+
+ // synchronously poll wait for a new storefile to appear (flush happened)
+ while (server.getStoreFileList(regName, FAMILY).size() == sfCount) {
+ Thread.sleep(40);
+ }
+ }
+ }
+
+ // override the config settings at the CF level and ensure priority
+ @Test(timeout = 60000)
+ public void testAdvancedConfigOverride() throws Exception {
+ /*
+ * Overall idea: (1) create 3 store files and issue a compaction. config's
+ * compaction.min == 3, so should work. (2) Increase the compaction.min
+ * toggle in the HTD to 5 and modify table. If we use the HTD value instead
+ * of the default config value, adding 3 files and issuing a compaction
+ * SHOULD NOT work (3) Decrease the compaction.min toggle in the HCD to 2
+ * and modify table. The CF schema should override the Table schema and now
+ * cause a minor compaction.
+ */
+ TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3);
+
+ String tableName = "testAdvancedConfigOverride";
+ byte[] TABLE = Bytes.toBytes(tableName);
+ HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
+ HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+ HConnection connection = HConnectionManager.getConnection(TEST_UTIL
+ .getConfiguration());
+
+ // Create 3 store files.
+ byte[] row = Bytes.toBytes(random.nextInt());
+ performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 100);
+
+ // Verify we have multiple store files.
+ HRegionLocation loc = hTable.getRegionLocation(row);
+ byte[] regionName = loc.getRegionInfo().getRegionName();
+ HRegionInterface server = connection.getHRegionConnection(loc
+ .getServerAddress());
+ assertTrue(server.getStoreFileList(regionName, FAMILY).size() > 1);
+
+ // Issue a compaction request
+ admin.compact(TABLE, FAMILY);
+
+ // poll wait for the compactions to happen
+ for (int i = 0; i < 10 * 1000 / 40; ++i) {
+ // The number of store files after compaction should be lesser.
+ loc = hTable.getRegionLocation(row);
+ if (!loc.getRegionInfo().isOffline()) {
+ regionName = loc.getRegionInfo().getRegionName();
+ server = connection.getHRegionConnection(loc.getServerAddress());
+ if (server.getStoreFileList(regionName, FAMILY).size() <= 1) {
+ break;
+ }
+ }
+ Thread.sleep(40);
+ }
+ // verify the compactions took place and that we didn't just time out
+ assertTrue(server.getStoreFileList(regionName, FAMILY).size() <= 1);
+
+ // change the compaction.min config option for this table to 5
+ HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor());
+ htd.setValue("hbase.hstore.compaction.min", String.valueOf(5));
+
+ admin.disableTable(TABLE);
+ admin.modifyTable(TABLE, htd);
+ admin.enableTable(TABLE);
+
+ // Create 3 more store files.
+ performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 10);
+
+ // Issue a compaction request
+ admin.compact(TABLE, FAMILY);
+
+ // This time, the compaction request should not happen
+ Thread.sleep(10 * 1000);
+ int sfCount = 0;
+ loc = hTable.getRegionLocation(row);
+ regionName = loc.getRegionInfo().getRegionName();
+ server = connection.getHRegionConnection(loc.getServerAddress());
+ sfCount = server.getStoreFileList(regionName, FAMILY).size();
+ assertTrue(sfCount > 1);
+
+ // change an individual CF's config option to 2 & online schema update
+ HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
+ hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2));
+ admin.alterTable(TABLE, new ArrayList<HColumnDescriptor>(), Lists
+ .newArrayList(new Pair<byte[], HColumnDescriptor>(FAMILY, hcd)),
+ new ArrayList<byte[]>());
+ Pair<Integer, Integer> st;
+ while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
+ LOG.debug(st.getFirst() + " regions left to update");
+ Thread.sleep(40);
+ }
+
+ // Issue a compaction request
+ admin.compact(TABLE, FAMILY);
+
+ // poll wait for the compactions to happen
+ for (int i = 0; i < 10 * 1000 / 40; ++i) {
+ loc = hTable.getRegionLocation(row);
+ regionName = loc.getRegionInfo().getRegionName();
+ try {
+ server = connection.getHRegionConnection(loc.getServerAddress());
+ if (server.getStoreFileList(regionName, FAMILY).size() < sfCount) {
+ break;
+ }
+ } catch (Exception e) {
+ LOG.debug("Waiting for region to come online: " + regionName);
+ }
+ Thread.sleep(40);
+ }
+ // verify the compaction took place and that we didn't just time out
+ assertTrue(server.getStoreFileList(regionName, FAMILY).size() < sfCount);
+
+ // Finally, ensure that we can remove a custom config value after we made it
+ hcd.setValue("hbase.hstore.compaction.min", null);
+ admin.alterTable(TABLE, new ArrayList<HColumnDescriptor>(), Lists
+ .newArrayList(new Pair<byte[], HColumnDescriptor>(FAMILY, hcd)),
+ new ArrayList<byte[]>());
+ while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
+ LOG.debug(st.getFirst() + " regions left to update");
+ Thread.sleep(40);
+ }
+ assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue(
+ "hbase.hstore.compaction.min"));
+ }
+
+}
Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java?rev=1311668&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundConfiguration.java Tue Apr 10 10:37:53 2012
@@ -0,0 +1,110 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.regionserver.CompoundConfiguration;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+
+import junit.framework.TestCase;
+
+
+public class TestCompoundConfiguration extends TestCase {
+ private Configuration baseConf;
+
+ @Override
+ protected void setUp() throws Exception {
+ baseConf = new Configuration();
+ baseConf.set("A", "1");
+ baseConf.setInt("B", 2);
+ baseConf.set("C", "3");
+ }
+
+ @Test
+ public void testBasicFunctionality() throws ClassNotFoundException {
+ CompoundConfiguration compoundConf = new CompoundConfiguration()
+ .add(baseConf);
+ assertEquals("1", compoundConf.get("A"));
+ assertEquals(2, compoundConf.getInt("B", 0));
+ assertEquals(3, compoundConf.getInt("C", 0));
+ assertEquals(0, compoundConf.getInt("D", 0));
+
+ assertEquals(CompoundConfiguration.class, compoundConf
+ .getClassByName(CompoundConfiguration.class.getName()));
+ try {
+ compoundConf.getClassByName("bad_class_name");
+ fail("Trying to load bad_class_name should throw an exception");
+ } catch (ClassNotFoundException e) {
+ // win!
+ }
+ }
+
+ @Test
+ public void testWithConfig() {
+ Configuration conf = new Configuration();
+ conf.set("B", "2b");
+ conf.set("C", "33");
+ conf.set("D", "4");
+
+ CompoundConfiguration compoundConf = new CompoundConfiguration()
+ .add(baseConf)
+ .add(conf);
+ assertEquals("1", compoundConf.get("A"));
+ assertEquals("2b", compoundConf.get("B"));
+ assertEquals(33, compoundConf.getInt("C", 0));
+ assertEquals("4", compoundConf.get("D"));
+ assertEquals(4, compoundConf.getInt("D", 0));
+ assertNull(compoundConf.get("E"));
+ assertEquals(6, compoundConf.getInt("F", 6));
+ }
+
+ private ImmutableBytesWritable strToIbw(String s) {
+ return new ImmutableBytesWritable(Bytes.toBytes(s));
+ }
+
+ @Test
+ public void testWithIbwMap() {
+ Map<ImmutableBytesWritable, ImmutableBytesWritable> map =
+ new HashMap<ImmutableBytesWritable, ImmutableBytesWritable>();
+ map.put(strToIbw("B"), strToIbw("2b"));
+ map.put(strToIbw("C"), strToIbw("33"));
+ map.put(strToIbw("D"), strToIbw("4"));
+ // unlike config, note that IBW Maps can accept null values
+ map.put(strToIbw("G"), null);
+
+ CompoundConfiguration compoundConf = new CompoundConfiguration()
+ .add(baseConf)
+ .add(map);
+ assertEquals("1", compoundConf.get("A"));
+ assertEquals("2b", compoundConf.get("B"));
+ assertEquals(33, compoundConf.getInt("C", 0));
+ assertEquals("4", compoundConf.get("D"));
+ assertEquals(4, compoundConf.getInt("D", 0));
+ assertNull(compoundConf.get("E"));
+ assertEquals(6, compoundConf.getInt("F", 6));
+ assertNull(compoundConf.get("G"));
+ }
+
+}