You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by br...@apache.org on 2008/02/13 20:30:30 UTC
svn commit: r627573 - in /hadoop/hbase/trunk: CHANGES.txt
src/java/org/apache/hadoop/hbase/HConnectionManager.java
src/java/org/apache/hadoop/hbase/util/SoftSortedMap.java
src/test/org/apache/hadoop/hbase/util/SoftSortedMapTest.java
Author: bryanduxbury
Date: Wed Feb 13 11:30:26 2008
New Revision: 627573
URL: http://svn.apache.org/viewvc?rev=627573&view=rev
Log:
HBASE-407 Client should cache region locations in an LRU structure
Added:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/SoftSortedMap.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/SoftSortedMapTest.java
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConnectionManager.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=627573&r1=627572&r2=627573&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Wed Feb 13 11:30:26 2008
@@ -45,8 +45,11 @@
HBASE-436 website: http://hadoop.apache.org/hbase
HBASE-417 Factor TableOperation and subclasses into separate files from
HMaster (Bryan Duxbury via Stack)
+
HBASE-440 Add optional log roll interval so that log files are garbage
collected
+ HBASE-407 Keep HRegionLocation information in LRU structure
+ (Bryan Duxbury)
Branch 0.1
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConnectionManager.java?rev=627573&r1=627572&r2=627573&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConnectionManager.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConnectionManager.java Wed Feb 13 11:30:26 2008
@@ -20,15 +20,12 @@
package org.apache.hadoop.hbase;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
-import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
-
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.logging.Log;
@@ -41,6 +38,7 @@
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.hbase.master.HMasterInterface;
+import org.apache.hadoop.hbase.util.SoftSortedMap;
/**
* A non-instantiable class that manages connections to multiple tables in
@@ -115,7 +113,9 @@
private HRegionLocation rootRegionLocation;
- private Map<Text, SortedMap<Text, HRegionLocation>> cachedRegionLocations;
+ private Map<Text, SoftSortedMap<Text, HRegionLocation>>
+ cachedRegionLocations = new ConcurrentHashMap<Text,
+ SoftSortedMap<Text, HRegionLocation>>();;
/**
* constructor
@@ -144,9 +144,6 @@
this.master = null;
this.masterChecked = false;
-
- this.cachedRegionLocations =
- new ConcurrentHashMap<Text, SortedMap<Text, HRegionLocation>>();
this.servers = new ConcurrentHashMap<String, HRegionInterface>();
}
@@ -489,12 +486,12 @@
*/
private HRegionLocation getCachedLocation(Text tableName, Text row) {
// find the map of cached locations for this table
- SortedMap<Text, HRegionLocation> tableLocations =
+ SoftSortedMap<Text, HRegionLocation> tableLocations =
cachedRegionLocations.get(tableName);
// if tableLocations for this table isn't built yet, make one
if (tableLocations == null) {
- tableLocations = new TreeMap<Text, HRegionLocation>();
+ tableLocations = new SoftSortedMap<Text, HRegionLocation>();
cachedRegionLocations.put(tableName, tableLocations);
}
@@ -514,7 +511,7 @@
// cut the cache so that we only get the part that could contain
// regions that match our key
- SortedMap<Text, HRegionLocation> matchingRegions =
+ SoftSortedMap<Text, HRegionLocation> matchingRegions =
tableLocations.headMap(row);
// if that portion of the map is empty, then we're done. otherwise,
@@ -523,7 +520,7 @@
if (!matchingRegions.isEmpty()) {
HRegionLocation possibleRegion =
matchingRegions.get(matchingRegions.lastKey());
-
+
Text endKey = possibleRegion.getRegionInfo().getEndKey();
// make sure that the end key is greater than the row we're looking
@@ -551,12 +548,12 @@
*/
private void deleteCachedLocation(Text tableName, Text row){
// find the map of cached locations for this table
- SortedMap<Text, HRegionLocation> tableLocations =
+ SoftSortedMap<Text, HRegionLocation> tableLocations =
cachedRegionLocations.get(tableName);
// if tableLocations for this table isn't built yet, make one
if (tableLocations == null) {
- tableLocations = new TreeMap<Text, HRegionLocation>();
+ tableLocations = new SoftSortedMap<Text, HRegionLocation>();
cachedRegionLocations.put(tableName, tableLocations);
}
@@ -565,7 +562,7 @@
if (!tableLocations.isEmpty()) {
// cut the cache so that we only get the part that could contain
// regions that match our key
- SortedMap<Text, HRegionLocation> matchingRegions =
+ SoftSortedMap<Text, HRegionLocation> matchingRegions =
tableLocations.headMap(row);
// if that portion of the map is empty, then we're done. otherwise,
@@ -599,12 +596,12 @@
Text startKey = location.getRegionInfo().getStartKey();
// find the map of cached locations for this table
- SortedMap<Text, HRegionLocation> tableLocations =
+ SoftSortedMap<Text, HRegionLocation> tableLocations =
cachedRegionLocations.get(tableName);
// if tableLocations for this table isn't built yet, make one
if (tableLocations == null) {
- tableLocations = new TreeMap<Text, HRegionLocation>();
+ tableLocations = new SoftSortedMap<Text, HRegionLocation>();
cachedRegionLocations.put(tableName, tableLocations);
}
Added: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/SoftSortedMap.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/SoftSortedMap.java?rev=627573&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/SoftSortedMap.java (added)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/SoftSortedMap.java Wed Feb 13 11:30:26 2008
@@ -0,0 +1,187 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.lang.ref.SoftReference;
+import java.lang.ref.ReferenceQueue;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.Comparator;
+import java.util.Collection;
+import java.util.ArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * A SortedMap implementation that uses SoftReferences internally to make it
+ * play well with the GC when in a low-memory situation.
+ */
+public class SoftSortedMap<K,V> implements SortedMap<K,V> {
+ protected static final Log LOG = LogFactory.getLog(SoftSortedMap.class);
+
+ protected SortedMap<K, SoftValue<K,V>> internalMap =
+ new TreeMap<K, SoftValue<K,V>>();
+
+ protected ReferenceQueue referenceQueue = new ReferenceQueue();
+
+ /** Constructor */
+ public SoftSortedMap() {}
+
+ /** For headMap and tailMap support */
+ private SoftSortedMap(SortedMap<K,SoftValue<K,V>> original) {
+ internalMap = original;
+ }
+
+ /* Client methods */
+ public V put(K key, V value) {
+ checkReferences();
+ SoftValue<K,V> oldValue =
+ internalMap.put(key, new SoftValue<K,V>(key, value, referenceQueue));
+ return oldValue == null ? null : oldValue.get();
+ }
+
+ public void putAll(Map map) {
+ throw new RuntimeException("Not implemented");
+ }
+
+ public V get(Object key) {
+ checkReferences();
+ SoftValue<K,V> value = internalMap.get(key);
+ return value == null ? null : value.get();
+ }
+
+ public V remove(Object key) {
+ checkReferences();
+ SoftValue<K,V> value = internalMap.remove(key);
+ return value == null ? null : value.get();
+ }
+
+ public boolean containsKey(Object key) {
+ checkReferences();
+ return internalMap.containsKey(key);
+ }
+
+ public boolean containsValue(Object value) {
+ checkReferences();
+ return internalMap.containsValue(value);
+ }
+
+ public K firstKey() {
+ checkReferences();
+ return internalMap.firstKey();
+ }
+
+ public K lastKey() {
+ checkReferences();
+ return internalMap.lastKey();
+ }
+
+ public SoftSortedMap<K,V> headMap(K key) {
+ checkReferences();
+ return new SoftSortedMap<K,V>(internalMap.headMap(key));
+ }
+
+ public SoftSortedMap<K,V> tailMap(K key) {
+ checkReferences();
+ return new SoftSortedMap<K,V>(internalMap.tailMap(key));
+ }
+
+ public SoftSortedMap<K,V> subMap(K fromKey, K toKey) {
+ checkReferences();
+ return new SoftSortedMap<K,V>(internalMap.subMap(fromKey, toKey));
+ }
+
+ public boolean isEmpty() {
+ checkReferences();
+ return internalMap.isEmpty();
+ }
+
+ public int size() {
+ checkReferences();
+ return internalMap.size();
+ }
+
+ public void clear() {
+ internalMap.clear();
+ }
+
+ public Set<K> keySet() {
+ return internalMap.keySet();
+ }
+
+ public Comparator comparator() {
+ return internalMap.comparator();
+ }
+
+ public Set<Map.Entry<K,V>> entrySet() {
+ Set<Map.Entry<K, SoftValue<K,V>>> entries = internalMap.entrySet();
+ Set<Map.Entry<K, V>> real_entries = new TreeSet<Map.Entry<K,V>>();
+ for(Map.Entry<K, SoftValue<K,V>> entry : entries) {
+ real_entries.add(entry.getValue());
+ }
+ return real_entries;
+ }
+
+ public Collection<V> values() {
+ checkReferences();
+ Collection<SoftValue<K,V>> softValues = internalMap.values();
+ ArrayList<V> hardValues = new ArrayList<V>();
+ for(SoftValue<K,V> softValue : softValues) {
+ hardValues.add(softValue.get());
+ }
+ return hardValues;
+ }
+
+ /**
+ * Check the reference queue and delete anything that has since gone away
+ */
+ private void checkReferences() {
+ SoftValue<K,V> sv;
+ while((sv = (SoftValue<K,V>)referenceQueue.poll()) != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Reference for key " + sv.key.toString() + " has been cleared.");
+ }
+ internalMap.remove(sv.key);
+ }
+ }
+
+ /**
+ * A SoftReference derivative so that we can track down what keys to remove.
+ */
+ private class SoftValue<K2, V2> extends SoftReference<V2> implements Map.Entry<K2,V2> {
+ K2 key;
+
+ SoftValue(K2 key, V2 value, ReferenceQueue queue) {
+ super(value, queue);
+ this.key = key;
+ }
+
+ public K2 getKey() {return key;}
+ public V2 getValue() {return get();}
+
+ public V2 setValue(V2 value) {
+ throw new RuntimeException("Not implemented");
+ }
+ }
+}
\ No newline at end of file
Added: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/SoftSortedMapTest.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/SoftSortedMapTest.java?rev=627573&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/SoftSortedMapTest.java (added)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/SoftSortedMapTest.java Wed Feb 13 11:30:26 2008
@@ -0,0 +1,41 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.util;
+
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+public class SoftSortedMapTest {
+ private static void testMap(SortedMap<Integer, Integer> map) {
+ System.out.println("Testing " + map.getClass());
+ for(int i = 0; i < 1000000; i++) {
+ map.put(new Integer(i), new Integer(i));
+ }
+ System.out.println(map.size());
+ byte[] block = new byte[849*1024*1024]; // 10 MB
+ System.out.println(map.size());
+ }
+
+ public static void main(String[] args) {
+ testMap(new SoftSortedMap<Integer, Integer>());
+ testMap(new TreeMap<Integer, Integer>());
+ }
+}
\ No newline at end of file
Re: svn commit: r627573 - in /hadoop/hbase/trunk: CHANGES.txt src/java/org/apache/hadoop/hbase/HConnectionManager.java
src/java/org/apache/hadoop/hbase/util/SoftSortedMap.java src/test/org/apache/hadoop/hbase/util/SoftSortedMapTest.java
Posted by stack <st...@duboce.net>.
bryanduxbury@apache.org wrote:
> ...
>
> Modified: hadoop/hbase/trunk/CHANGES.txt
> URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=627573&r1=627572&r2=627573&view=diff
> ==============================================================================
> --- hadoop/hbase/trunk/CHANGES.txt (original)
> +++ hadoop/hbase/trunk/CHANGES.txt Wed Feb 13 11:30:26 2008
> @@ -45,8 +45,11 @@
> HBASE-436 website: http://hadoop.apache.org/hbase
> HBASE-417 Factor TableOperation and subclasses into separate files from
> HMaster (Bryan Duxbury via Stack)
> +
> HBASE-440 Add optional log roll interval so that log files are garbage
> collected
> + HBASE-407 Keep HRegionLocation information in LRU structure
> + (Bryan Duxbury)
>
General pattern is not to accredit patches (i.e. no need of the '(Bryan
Duxbury)' in the above).
St.Ack