You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2008/12/22 03:42:50 UTC
svn commit: r728571 [2/2] - in /hadoop/hbase/branches/0.19_on_hadoop_0.18:
./ conf/ src/java/org/apache/hadoop/hbase/
src/java/org/apache/hadoop/hbase/client/
src/java/org/apache/hadoop/hbase/io/ src/java/org/apache/hadoop/hbase/ipc/
src/java/org/apach...
Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/Migrate.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/Migrate.java?rev=728571&r1=728570&r2=728571&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/Migrate.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/Migrate.java Sun Dec 21 18:42:49 2008
@@ -191,8 +191,8 @@
throw new IOException(msg);
}
- migrateTo5();
-
+ migrate4To6();
+
if (!readOnly) {
// Set file system version
LOG.info("Setting file system version.");
@@ -208,16 +208,19 @@
}
}
- // Move the fileystem version from 4 to 5.
+ // Move the fileystem version from 4 to 6.
// In here we rewrite the catalog table regions so they keep 10 versions
// instead of 1.
- private void migrateTo5() throws IOException {
+ private void migrate4To6() throws IOException {
if (this.readOnly && this.migrationNeeded) {
return;
}
final MetaUtils utils = new MetaUtils(this.conf);
try {
+ // These two operations are effectively useless. -ROOT- is hardcode,
+ // at least until hbase 0.20.0 when we store it out in ZK.
updateVersions(utils.getRootRegion().getRegionInfo());
+ enableBlockCache(utils.getRootRegion().getRegionInfo());
// Scan the root region
utils.scanRootRegion(new MetaUtils.ScannerListener() {
public boolean processRow(HRegionInfo info)
@@ -227,6 +230,7 @@
return false;
}
updateVersions(utils.getRootRegion(), info);
+ enableBlockCache(utils.getRootRegion(), info);
return true;
}
});
@@ -236,8 +240,41 @@
}
/*
- * Move from old pre-v5 hregioninfo to current HRegionInfo
- * Persist back into <code>r</code>
+ * Enable blockcaching on catalog tables.
+ * @param mr
+ * @param oldHri
+ */
+ void enableBlockCache(HRegion mr, HRegionInfo oldHri)
+ throws IOException {
+ if (!enableBlockCache(oldHri)) {
+ return;
+ }
+ BatchUpdate b = new BatchUpdate(oldHri.getRegionName());
+ b.put(HConstants.COL_REGIONINFO, Writables.getBytes(oldHri));
+ mr.batchUpdate(b);
+ LOG.info("Enabled blockcache on " + oldHri.getRegionNameAsString());
+ }
+
+ /*
+ * @param hri Update versions.
+ * @param true if we changed value
+ */
+ private boolean enableBlockCache(final HRegionInfo hri) {
+ boolean result = false;
+ HColumnDescriptor hcd =
+ hri.getTableDesc().getFamily(HConstants.COLUMN_FAMILY);
+ if (hcd == null) {
+ LOG.info("No info family in: " + hri.getRegionNameAsString());
+ return result;
+ }
+ // Set blockcache enabled.
+ hcd.setBlockCacheEnabled(true);
+ return true;
+ }
+
+
+ /*
+ * Update versions kept in historian.
* @param mr
* @param oldHri
*/
@@ -251,7 +288,7 @@
mr.batchUpdate(b);
LOG.info("Upped versions on " + oldHri.getRegionNameAsString());
}
-
+
/*
* @param hri Update versions.
* @param true if we changed value
Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/ReferenceQueueUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/ReferenceQueueUtil.java?rev=728571&r1=728570&r2=728571&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/ReferenceQueueUtil.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/ReferenceQueueUtil.java Sun Dec 21 18:42:49 2008
@@ -1,65 +0,0 @@
-/**
- * Copyright 2008 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.util;
-
-import java.lang.ref.ReferenceQueue;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-/**
- * java.lang.ref.ReferenceQueue utility class.
- * @param <K>
- * @param <V>
- */
-class ReferenceQueueUtil<K,V> {
- private final Log LOG = LogFactory.getLog(this.getClass());
- private final ReferenceQueue rq = new ReferenceQueue();
- private final Map<K,V> map;
-
- private ReferenceQueueUtil() {
- this(null);
- }
-
- ReferenceQueueUtil(final Map<K,V> m) {
- super();
- this.map = m;
- }
-
- public ReferenceQueue getReferenceQueue() {
- return rq;
- }
-
- /**
- * Check the reference queue and delete anything that has since gone away
- */
- @SuppressWarnings("unchecked")
- void checkReferences() {
- int i = 0;
- for (Object obj = null; (obj = this.rq.poll()) != null;) {
- i++;
- this.map.remove(((SoftValue<K,V>)obj).getKey());
- }
- if (i > 0 && LOG.isDebugEnabled()) {
- LOG.debug("" + i + " reference(s) cleared.");
- }
- }
-}
\ No newline at end of file
Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/SoftValueMap.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/SoftValueMap.java?rev=728571&r1=728570&r2=728571&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/SoftValueMap.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/SoftValueMap.java Sun Dec 21 18:42:49 2008
@@ -19,6 +19,7 @@
*/
package org.apache.hadoop.hbase.util;
+import java.lang.ref.ReferenceQueue;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -35,13 +36,34 @@
public class SoftValueMap<K,V> implements Map<K,V> {
private final Map<K, SoftValue<K,V>> internalMap =
new HashMap<K, SoftValue<K,V>>();
- private final ReferenceQueueUtil<K,SoftValue<K,V>> rq =
- new ReferenceQueueUtil<K,SoftValue<K,V>>(this.internalMap);
+ private final ReferenceQueue<?> rq;
+ public SoftValueMap() {
+ this(new ReferenceQueue());
+ }
+
+ public SoftValueMap(final ReferenceQueue<?> rq) {
+ this.rq = rq;
+ }
+
+ /**
+ * Checks soft references and cleans any that have been placed on
+ * ReferenceQueue.
+ * @return How many references cleared.
+ */
+ public int checkReferences() {
+ int i = 0;
+ for (Object obj = null; (obj = this.rq.poll()) != null;) {
+ i++;
+ this.internalMap.remove(((SoftValue<K,V>)obj).getKey());
+ }
+ return i;
+ }
+
public V put(K key, V value) {
- this.rq.checkReferences();
+ checkReferences();
SoftValue<K,V> oldValue = this.internalMap.put(key,
- new SoftValue<K,V>(key, value, this.rq.getReferenceQueue()));
+ new SoftValue<K,V>(key, value, this.rq));
return oldValue == null ? null : oldValue.get();
}
@@ -51,7 +73,7 @@
}
public V get(Object key) {
- this.rq.checkReferences();
+ checkReferences();
SoftValue<K,V> value = this.internalMap.get(key);
if (value == null) {
return null;
@@ -64,13 +86,13 @@
}
public V remove(Object key) {
- this.rq.checkReferences();
+ checkReferences();
SoftValue<K,V> value = this.internalMap.remove(key);
return value == null ? null : value.get();
}
public boolean containsKey(Object key) {
- this.rq.checkReferences();
+ checkReferences();
return this.internalMap.containsKey(key);
}
@@ -81,27 +103,27 @@
}
public boolean isEmpty() {
- this.rq.checkReferences();
+ checkReferences();
return this.internalMap.isEmpty();
}
public int size() {
- this.rq.checkReferences();
+ checkReferences();
return this.internalMap.size();
}
public void clear() {
- this.rq.checkReferences();
+ checkReferences();
this.internalMap.clear();
}
public Set<K> keySet() {
- this.rq.checkReferences();
+ checkReferences();
return this.internalMap.keySet();
}
public Set<Map.Entry<K,V>> entrySet() {
- this.rq.checkReferences();
+ checkReferences();
Set<Map.Entry<K, SoftValue<K,V>>> entries = this.internalMap.entrySet();
Set<Map.Entry<K, V>> real_entries = new HashSet<Map.Entry<K,V>>();
for(Map.Entry<K, SoftValue<K,V>> entry : entries) {
@@ -111,7 +133,7 @@
}
public Collection<V> values() {
- this.rq.checkReferences();
+ checkReferences();
Collection<SoftValue<K,V>> softValues = this.internalMap.values();
ArrayList<V> hardValues = new ArrayList<V>();
for(SoftValue<K,V> softValue : softValues) {
Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java?rev=728571&r1=728570&r2=728571&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java Sun Dec 21 18:42:49 2008
@@ -19,6 +19,7 @@
*/
package org.apache.hadoop.hbase.util;
+import java.lang.ref.ReferenceQueue;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
@@ -38,7 +39,7 @@
*/
public class SoftValueSortedMap<K,V> implements SortedMap<K,V> {
private final SortedMap<K, SoftValue<K,V>> internalMap;
- private final ReferenceQueueUtil<K,SoftValue<K,V>> rq;
+ private final ReferenceQueue rq = new ReferenceQueue();
/** Constructor */
public SoftValueSortedMap() {
@@ -56,13 +57,27 @@
/** For headMap and tailMap support */
private SoftValueSortedMap(SortedMap<K,SoftValue<K,V>> original) {
this.internalMap = original;
- this.rq = new ReferenceQueueUtil<K,SoftValue<K,V>>(this.internalMap);
}
-
+
+ /**
+ * Checks soft references and cleans any that have been placed on
+ * ReferenceQueue. Call if get/put etc. are not called regularly.
+ * Internally these call checkReferences on each access.
+ * @return How many references cleared.
+ */
+ public int checkReferences() {
+ int i = 0;
+ for (Object obj = null; (obj = this.rq.poll()) != null;) {
+ i++;
+ this.internalMap.remove(((SoftValue<K,V>)obj).getKey());
+ }
+ return i;
+ }
+
public V put(K key, V value) {
- this.rq.checkReferences();
+ checkReferences();
SoftValue<K,V> oldValue = this.internalMap.put(key,
- new SoftValue<K,V>(key, value, this.rq.getReferenceQueue()));
+ new SoftValue<K,V>(key, value, this.rq));
return oldValue == null ? null : oldValue.get();
}
@@ -72,7 +87,7 @@
}
public V get(Object key) {
- this.rq.checkReferences();
+ checkReferences();
SoftValue<K,V> value = this.internalMap.get(key);
if (value == null) {
return null;
@@ -85,13 +100,13 @@
}
public V remove(Object key) {
- this.rq.checkReferences();
+ checkReferences();
SoftValue<K,V> value = this.internalMap.remove(key);
return value == null ? null : value.get();
}
public boolean containsKey(Object key) {
- this.rq.checkReferences();
+ checkReferences();
return this.internalMap.containsKey(key);
}
@@ -102,47 +117,47 @@
}
public K firstKey() {
- this.rq.checkReferences();
+ checkReferences();
return internalMap.firstKey();
}
public K lastKey() {
- this.rq.checkReferences();
+ checkReferences();
return internalMap.lastKey();
}
public SoftValueSortedMap<K,V> headMap(K key) {
- this.rq.checkReferences();
+ checkReferences();
return new SoftValueSortedMap<K,V>(this.internalMap.headMap(key));
}
public SoftValueSortedMap<K,V> tailMap(K key) {
- this.rq.checkReferences();
+ checkReferences();
return new SoftValueSortedMap<K,V>(this.internalMap.tailMap(key));
}
public SoftValueSortedMap<K,V> subMap(K fromKey, K toKey) {
- this.rq.checkReferences();
+ checkReferences();
return new SoftValueSortedMap<K,V>(this.internalMap.subMap(fromKey, toKey));
}
public boolean isEmpty() {
- this.rq.checkReferences();
+ checkReferences();
return this.internalMap.isEmpty();
}
public int size() {
- this.rq.checkReferences();
+ checkReferences();
return this.internalMap.size();
}
public void clear() {
- this.rq.checkReferences();
+ checkReferences();
this.internalMap.clear();
}
public Set<K> keySet() {
- this.rq.checkReferences();
+ checkReferences();
return this.internalMap.keySet();
}
@@ -152,7 +167,7 @@
}
public Set<Map.Entry<K,V>> entrySet() {
- this.rq.checkReferences();
+ checkReferences();
Set<Map.Entry<K, SoftValue<K,V>>> entries = this.internalMap.entrySet();
Set<Map.Entry<K, V>> real_entries = new TreeSet<Map.Entry<K,V>>();
for(Map.Entry<K, SoftValue<K,V>> entry : entries) {
@@ -162,7 +177,7 @@
}
public Collection<V> values() {
- this.rq.checkReferences();
+ checkReferences();
Collection<SoftValue<K,V>> softValues = this.internalMap.values();
ArrayList<V> hardValues = new ArrayList<V>();
for(SoftValue<K,V> softValue : softValues) {
Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java?rev=728571&r1=728570&r2=728571&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java Sun Dec 21 18:42:49 2008
@@ -110,8 +110,28 @@
* @return the region server that was stopped
*/
public LocalHBaseCluster.RegionServerThread stopRegionServer(int serverNumber) {
- LocalHBaseCluster.RegionServerThread server = hbaseCluster.getRegionServers().get(serverNumber);
+ return stopRegionServer(serverNumber, true);
+ }
+
+ /**
+ * Shut down the specified region server cleanly
+ *
+ * @param serverNumber Used as index into a list.
+ * @param shutdownFS True is we are to shutdown the filesystem as part of this
+ * regionserver's shutdown. Usually we do but you do not want to do this if
+ * you are running multiple regionservers in a test and you shut down one
+ * before end of the test.
+ * @return the region server that was stopped
+ */
+ public LocalHBaseCluster.RegionServerThread stopRegionServer(int serverNumber,
+ final boolean shutdownFS) {
+ LocalHBaseCluster.RegionServerThread server =
+ hbaseCluster.getRegionServers().get(serverNumber);
LOG.info("Stopping " + server.toString());
+ if (!shutdownFS) {
+ // Stop the running of the hdfs shutdown thread in tests.
+ server.getRegionServer().setHDFSShutdownThreadOnExit(null);
+ }
server.getRegionServer().stop();
return server;
}
Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/TestRegionRebalancing.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/TestRegionRebalancing.java?rev=728571&r1=728570&r2=728571&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/TestRegionRebalancing.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/TestRegionRebalancing.java Sun Dec 21 18:42:49 2008
@@ -113,7 +113,7 @@
// kill a region server - total of 2
LOG.debug("Killing the 3rd region server.");
- cluster.stopRegionServer(2);
+ cluster.stopRegionServer(2, false);
cluster.waitOnRegionServer(2);
assertRegionsAreBalanced();
Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java?rev=728571&r1=728570&r2=728571&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java Sun Dec 21 18:42:49 2008
@@ -40,6 +40,11 @@
private static final byte[] key_mmi = Bytes.toBytes("mmi");
private static final byte[] key_ssm = Bytes.toBytes("ssm");
+ protected void setUp() throws Exception {
+ super.setUp();
+ this.conf.setInt("hbase.io.index.interval", 32);
+ }
+
/**
* the test
* @throws IOException
Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/regionserver/TestCompaction.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/regionserver/TestCompaction.java?rev=728571&r1=728570&r2=728571&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/regionserver/TestCompaction.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/regionserver/TestCompaction.java Sun Dec 21 18:42:49 2008
@@ -57,7 +57,7 @@
// Set cache flush size to 1MB
conf.setInt("hbase.hregion.memcache.flush.size", 1024*1024);
- conf.setInt("hbase.hregion.memcache.block.multiplier", 2);
+ conf.setInt("hbase.hregion.memcache.block.multiplier", 10);
this.cluster = null;
}
Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/regionserver/transactional/DisabledTestHLogRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/regionserver/transactional/DisabledTestHLogRecovery.java?rev=728571&r1=728570&r2=728571&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/regionserver/transactional/DisabledTestHLogRecovery.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/regionserver/transactional/DisabledTestHLogRecovery.java Sun Dec 21 18:42:49 2008
@@ -192,7 +192,7 @@
this.cluster.abortRegionServer(server);
} else {
- this.cluster.stopRegionServer(server);
+ this.cluster.stopRegionServer(server, false);
}
LOG.info(this.cluster.waitOnRegionServer(server) + " has been "
+ (abort ? "aborted" : "shut down"));
Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/webapps/master/master.jsp
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/webapps/master/master.jsp?rev=728571&r1=728570&r2=728571&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/webapps/master/master.jsp (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/webapps/master/master.jsp Sun Dec 21 18:42:49 2008
@@ -25,7 +25,7 @@
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
- <meta http-equiv="refresh" content="30"/>
+ <meta http-equiv="refresh" content="300"/>
<title>HBase Master: <%= master.getMasterAddress().getHostname()%>:<%= master.getMasterAddress().getPort() %></title>
<link rel="stylesheet" type="text/css" href="/static/hbase.css" />
</head>
Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/webapps/regionserver/regionserver.jsp
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/webapps/regionserver/regionserver.jsp?rev=728571&r1=728570&r2=728571&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/webapps/regionserver/regionserver.jsp (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/webapps/regionserver/regionserver.jsp Sun Dec 21 18:42:49 2008
@@ -20,7 +20,7 @@
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
- <meta http-equiv="refresh" content="30"/>
+ <meta http-equiv="refresh" content="300"/>
<title>HBase Region Server: <%= serverInfo.getServerAddress().getHostname() %>:<%= serverInfo.getServerAddress().getPort() %></title>
<link rel="stylesheet" type="text/css" href="/static/hbase.css" />
</head>