You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ji...@apache.org on 2008/12/03 01:48:00 UTC
svn commit: r722704 - in /hadoop/hbase/trunk: ./
src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/master/
src/java/org/apache/hadoop/hbase/regionserver/
Author: jimk
Date: Tue Dec 2 16:47:59 2008
New Revision: 722704
URL: http://svn.apache.org/viewvc?rev=722704&view=rev
Log:
HBASE-927 We don't recover if HRS hosting -ROOT-/.META. goes down
Added:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/NotAllMetaRegionsOnlineException.java
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HServerLoad.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/DeleteColumn.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaScanner.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ModifyColumn.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionManager.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionServerOperation.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ServerManager.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=722704&r1=722703&r2=722704&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Tue Dec 2 16:47:59 2008
@@ -89,6 +89,7 @@
HBASE-1037 Some test cases failing on Windows/Cygwin but not UNIX/Linux
HBASE-1041 Migration throwing NPE
HBASE-1042 OOME but we don't abort
+ HBASE-927 We don't recover if HRS hosting -ROOT-/.META. goes down
IMPROVEMENTS
HBASE-901 Add a limit to key length, check key and value length on client side
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HServerLoad.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HServerLoad.java?rev=722704&r1=722703&r2=722704&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HServerLoad.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HServerLoad.java Tue Dec 2 16:47:59 2008
@@ -23,12 +23,7 @@
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Strings;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
@@ -36,7 +31,7 @@
/**
* This class encapsulates metrics for determining the load on a HRegionServer
*/
-@SuppressWarnings("unused")
+@SuppressWarnings("unchecked")
public class HServerLoad implements WritableComparable {
/** number of regions */
// could just use regionLoad.size() but master.RegionManager likes to play
@@ -138,7 +133,7 @@
}
/**
- * @param storefiles the number of stores
+ * @param stores the number of stores
*/
public void setStores(int stores) {
this.stores = stores;
@@ -303,10 +298,6 @@
return numberOfRegions;
}
- public Collection<RegionLoad> getRegionLoad() {
- return Collections.unmodifiableCollection(regionLoad);
- }
-
/**
* @return the numberOfRequests
*/
@@ -383,13 +374,14 @@
}
/**
- -* @param name
- -* @param stores
- -* @param storefiles
- -* @param memcacheSizeMB
- -* @param storefileIndexSizeMB
+ * @param name
+ * @param stores
+ * @param storefiles
+ * @param memcacheSizeMB
+ * @param storefileIndexSizeMB
* @deprecated Use {@link #addRegionInfo(RegionLoad)}
*/
+ @Deprecated
public void addRegionInfo(final byte[] name, final int stores,
final int storefiles, final int memcacheSizeMB,
final int storefileIndexSizeMB) {
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/DeleteColumn.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/DeleteColumn.java?rev=722704&r1=722703&r2=722704&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/DeleteColumn.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/DeleteColumn.java Tue Dec 2 16:47:59 2008
@@ -22,9 +22,7 @@
import java.io.IOException;
import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.fs.FileSystem; //TODO: remove
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.regionserver.HStoreFile; //TODO: remove
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java?rev=722704&r1=722703&r2=722704&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/HMaster.java Tue Dec 2 16:47:59 2008
@@ -125,7 +125,7 @@
final int metaRescanInterval;
// A Sleeper that sleeps for threadWakeFrequency
- protected final Sleeper sleeper;
+ private final Sleeper sleeper;
// Default access so accesible from unit tests. MASTER is name of the webapp
// and the attribute name used stuffing this instance into web context.
@@ -590,27 +590,14 @@
if (!isMasterRunning()) {
throw new MasterNotRunningException();
}
- HRegionInfo newRegion = new HRegionInfo(desc, null, null);
-
- for (int tries = 0; tries < numRetries; tries++) {
- try {
- // We can not access meta regions if they have not already been
- // assigned and scanned. If we timeout waiting, just shutdown.
- if (regionManager.waitForMetaRegionsOrClose()) {
- break;
- }
- createTable(newRegion);
- LOG.info("created table " + desc.getNameAsString());
- break;
- } catch (TableExistsException e) {
- throw e;
- } catch (IOException e) {
- if (tries == numRetries - 1) {
- throw RemoteExceptionHandler.checkIOException(e);
- }
- sleeper.sleep();
- }
+ // We can not create a table unless meta regions have already been
+ // assigned and scanned.
+ if (!regionManager.areAllMetaRegionsOnline()) {
+ throw new NotAllMetaRegionsOnlineException();
}
+ HRegionInfo newRegion = new HRegionInfo(desc, null, null);
+ createTable(newRegion);
+ LOG.info("created table " + desc.getNameAsString());
}
private synchronized void createTable(final HRegionInfo newRegion)
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaScanner.java?rev=722704&r1=722703&r2=722704&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaScanner.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaScanner.java Tue Dec 2 16:47:59 2008
@@ -61,7 +61,7 @@
private boolean scanOneMetaRegion(MetaRegion region) {
while (!master.closed.get() && !regionManager.isInitialRootScanComplete() &&
regionManager.getRootRegionLocation() == null) {
- master.sleeper.sleep();
+ sleep();
}
if (master.closed.get()) {
return false;
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ModifyColumn.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ModifyColumn.java?rev=722704&r1=722703&r2=722704&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ModifyColumn.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ModifyColumn.java Tue Dec 2 16:47:59 2008
@@ -19,7 +19,6 @@
*/
package org.apache.hadoop.hbase.master;
-import java.util.Map; //TODO: remove
import java.io.IOException;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
Added: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/NotAllMetaRegionsOnlineException.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/NotAllMetaRegionsOnlineException.java?rev=722704&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/NotAllMetaRegionsOnlineException.java (added)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/NotAllMetaRegionsOnlineException.java Tue Dec 2 16:47:59 2008
@@ -0,0 +1,44 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master;
+
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+
+/**
+ * Thrown when an operation requires the root and all meta regions to be online
+ */
+@SuppressWarnings("serial")
+public class NotAllMetaRegionsOnlineException extends DoNotRetryIOException {
+ /**
+ * default constructor
+ */
+ public NotAllMetaRegionsOnlineException() {
+ super();
+ }
+
+ /**
+ * @param message
+ */
+ public NotAllMetaRegionsOnlineException(String message) {
+ super(message);
+ }
+
+}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java?rev=722704&r1=722703&r2=722704&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java Tue Dec 2 16:47:59 2008
@@ -47,6 +47,7 @@
private final HServerAddress deadServer;
private final String deadServerName;
private final boolean rootRegionServer;
+ private boolean rootRegionReassigned = false;
private Path oldLogDir;
private boolean logSplit;
private boolean rootRescanned;
@@ -250,14 +251,19 @@
}
logSplit = true;
}
+
+ if (this.rootRegionServer && !this.rootRegionReassigned) {
+ // The server that died was serving the root region. Now that the log
+ // has been split, get it reassigned.
+ master.regionManager.reassignRootRegion();
+ // avoid multiple root region reassignment
+ this.rootRegionReassigned = true;
+ // When we call rootAvailable below, it will put us on the delayed
+ // to do queue to allow some time to pass during which the root
+ // region will hopefully get reassigned.
+ }
if (!rootAvailable()) {
- if (rootRegionServer) {
- // Get root region assigned now that log has been split and if the
- // dead server was serving the root region
- master.regionManager.reassignRootRegion();
- }
-
// Return true so that worker does not put this request back on the
// toDoQueue.
// rootAvailable() has already put it on the delayedToDoQueue
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=722704&r1=722703&r2=722704&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionManager.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionManager.java Tue Dec 2 16:47:59 2008
@@ -59,7 +59,7 @@
class RegionManager implements HConstants {
protected static final Log LOG = LogFactory.getLog(RegionManager.class);
- private volatile AtomicReference<HServerAddress> rootRegionLocation =
+ private AtomicReference<HServerAddress> rootRegionLocation =
new AtomicReference<HServerAddress>(null);
private volatile boolean safeMode = true;
@@ -165,7 +165,7 @@
}
void reassignRootRegion() {
- rootRegionLocation.set(null);
+ unsetRootRegion();
if (!master.shutdownRequested) {
unassignedRegions.put(HRegionInfo.ROOT_REGIONINFO, ZERO_L);
}
@@ -500,8 +500,13 @@
* Block until meta regions are online or we're shutting down.
* @return true if we found meta regions, false if we're closing.
*/
- public boolean waitForMetaRegionsOrClose() {
- return metaScannerThread.waitForMetaRegionsOrClose();
+ public boolean areAllMetaRegionsOnline() {
+ boolean result = false;
+ if (rootRegionLocation.get() != null &&
+ numberOfMetaRegions.get() == onlineMetaRegions.size()) {
+ result = true;
+ }
+ return result;
}
/**
@@ -530,16 +535,25 @@
* Get a set of all the meta regions that contain info about a given table.
* @param tableName Table you need to know all the meta regions for
* @return set of MetaRegion objects that contain the table
+ * @throws NotAllMetaRegionsOnlineException
*/
- public Set<MetaRegion> getMetaRegionsForTable(byte [] tableName) {
+ public Set<MetaRegion> getMetaRegionsForTable(byte [] tableName)
+ throws NotAllMetaRegionsOnlineException {
byte [] firstMetaRegion = null;
Set<MetaRegion> metaRegions = new HashSet<MetaRegion>();
if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
+ if (rootRegionLocation.get() == null) {
+ throw new NotAllMetaRegionsOnlineException(
+ Bytes.toString(HConstants.ROOT_TABLE_NAME));
+ }
metaRegions.add(new MetaRegion(rootRegionLocation.get(),
HRegionInfo.ROOT_REGIONINFO.getRegionName()));
} else {
+ if (!areAllMetaRegionsOnline()) {
+ throw new NotAllMetaRegionsOnlineException();
+ }
synchronized (onlineMetaRegions) {
if (onlineMetaRegions.size() == 1) {
firstMetaRegion = onlineMetaRegions.firstKey();
@@ -599,9 +613,9 @@
* @return list of MetaRegion objects
*/
public List<MetaRegion> getListOfOnlineMetaRegions() {
- List<MetaRegion> regions = new ArrayList<MetaRegion>();
+ List<MetaRegion> regions = null;
synchronized(onlineMetaRegions) {
- regions.addAll(onlineMetaRegions.values());
+ regions = new ArrayList<MetaRegion>(onlineMetaRegions.values());
}
return regions;
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionServerOperation.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionServerOperation.java?rev=722704&r1=722703&r2=722704&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionServerOperation.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionServerOperation.java Tue Dec 2 16:47:59 2008
@@ -69,12 +69,6 @@
protected boolean metaTableAvailable() {
boolean available = true;
- if (LOG.isDebugEnabled()) {
- LOG.debug("numberOfMetaRegions: " +
- master.regionManager.numMetaRegions() +
- ", onlineMetaRegions.size(): " +
- master.regionManager.numOnlineMetaRegions());
- }
if (master.regionManager.numMetaRegions() !=
master.regionManager.numOnlineMetaRegions()) {
// We can't proceed because not all of the meta regions are online.
@@ -83,6 +77,10 @@
// in the run queue, put this request on the delay queue to give
// other threads the opportunity to get the meta regions on-line.
if (LOG.isDebugEnabled()) {
+ LOG.debug("numberOfMetaRegions: " +
+ master.regionManager.numMetaRegions() +
+ ", onlineMetaRegions.size(): " +
+ master.regionManager.numOnlineMetaRegions());
LOG.debug("Requeuing because not all meta regions are online");
}
available = false;
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java?rev=722704&r1=722703&r2=722704&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java Tue Dec 2 16:47:59 2008
@@ -32,6 +32,7 @@
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.util.Sleeper;
/**
* Uses Callable pattern so that operations against meta regions do not need
@@ -39,6 +40,7 @@
*/
abstract class RetryableMetaOperation<T> implements Callable<T> {
protected final Log LOG = LogFactory.getLog(this.getClass());
+ protected final Sleeper sleeper;
protected final MetaRegion m;
protected final HMaster master;
@@ -47,6 +49,7 @@
protected RetryableMetaOperation(MetaRegion m, HMaster master) {
this.m = m;
this.master = master;
+ this.sleeper = new Sleeper(master.threadWakeFrequency, master.closed);
}
protected T doWithRetries()
@@ -89,7 +92,7 @@
} catch (Exception e) {
throw new RuntimeException(e);
}
- master.sleeper.sleep();
+ sleeper.sleep();
}
return null;
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=722704&r1=722703&r2=722704&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ServerManager.java Tue Dec 2 16:47:59 2008
@@ -127,7 +127,7 @@
HServerAddress root = master.getRootRegionLocation();
boolean rootServer = false;
if (root != null && root.equals(storedInfo.getServerAddress())) {
- master.regionManager.setRootRegionLocation(null);
+ master.regionManager.unsetRootRegion();
rootServer = true;
}
try {
@@ -293,17 +293,19 @@
serversToServerInfo.put(serverName, serverInfo);
HServerLoad load = serversToLoad.get(serverName);
- this.master.getMetrics().incrementRequests(load.getNumberOfRequests());
- if (load != null && !load.equals(serverInfo.getLoad())) {
- // We have previous information about the load on this server
- // and the load on this server has changed
- synchronized (loadToServers) {
- Set<String> servers = loadToServers.get(load);
-
- // Note that servers should never be null because loadToServers
- // and serversToLoad are manipulated in pairs
- servers.remove(serverName);
- loadToServers.put(load, servers);
+ if (load != null) {
+ this.master.getMetrics().incrementRequests(load.getNumberOfRequests());
+ if (!load.equals(serverInfo.getLoad())) {
+ // We have previous information about the load on this server
+ // and the load on this server has changed
+ synchronized (loadToServers) {
+ Set<String> servers = loadToServers.get(load);
+
+ // Note that servers should never be null because loadToServers
+ // and serversToLoad are manipulated in pairs
+ servers.remove(serverName);
+ loadToServers.put(load, servers);
+ }
}
}
@@ -715,19 +717,16 @@
}
}
deadServers.add(server);
- }
- synchronized (serversToServerInfo) {
- serversToServerInfo.notifyAll();
- }
-
- if (info != null) {
try {
master.toDoQueue.put(
new ProcessServerShutdown(master, info, rootServer));
} catch (InterruptedException e) {
- LOG.error("Insertion into toDoQueue was interrupted", e);
+ LOG.error("insert into toDoQueue was interrupted", e);
}
}
+ synchronized (serversToServerInfo) {
+ serversToServerInfo.notifyAll();
+ }
}
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java?rev=722704&r1=722703&r2=722704&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java Tue Dec 2 16:47:59 2008
@@ -699,85 +699,90 @@
LOG.debug("Splitting " + (i + 1) + " of " + logfiles.length + ": " +
logfiles[i].getPath());
}
- // Check for empty file.
- if (logfiles[i].getLen() <= 0) {
- LOG.info("Skipping " + logfiles[i].toString() +
- " because zero length");
- continue;
- }
+ // Check for possibly empty file. With appends, currently Hadoop reports
+ // a zero length even if the file has been sync'd. Revisit if
+ // HADOOP-4751 is committed.
+ boolean possiblyEmpty = logfiles[i].getLen() <= 0;
HLogKey key = new HLogKey();
HLogEdit val = new HLogEdit();
- SequenceFile.Reader in =
- new SequenceFile.Reader(fs, logfiles[i].getPath(), conf);
try {
- int count = 0;
- for (; in.next(key, val); count++) {
- byte [] tableName = key.getTablename();
- byte [] regionName = key.getRegionName();
- SequenceFile.Writer w = logWriters.get(regionName);
- if (w == null) {
- Path logfile = new Path(
- HRegion.getRegionDir(
- HTableDescriptor.getTableDir(rootDir, tableName),
- HRegionInfo.encodeRegionName(regionName)),
- HREGION_OLDLOGFILE_NAME);
- Path oldlogfile = null;
- SequenceFile.Reader old = null;
- if (fs.exists(logfile)) {
- LOG.warn("Old log file " + logfile +
- " already exists. Copying existing file to new file");
- oldlogfile = new Path(logfile.toString() + ".old");
- fs.rename(logfile, oldlogfile);
- old = new SequenceFile.Reader(fs, oldlogfile, conf);
- }
- w = SequenceFile.createWriter(fs, conf, logfile, HLogKey.class,
- HLogEdit.class, getCompressionType(conf));
- // Use copy of regionName; regionName object is reused inside in
- // HStoreKey.getRegionName so its content changes as we iterate.
- logWriters.put(regionName, w);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Creating new log file writer for path " + logfile +
- " and region " + Bytes.toString(regionName));
- }
-
- if (old != null) {
- // Copy from existing log file
- HLogKey oldkey = new HLogKey();
- HLogEdit oldval = new HLogEdit();
- for (; old.next(oldkey, oldval); count++) {
- if (LOG.isDebugEnabled() && count > 0 && count % 10000 == 0) {
- LOG.debug("Copied " + count + " edits");
+ SequenceFile.Reader in =
+ new SequenceFile.Reader(fs, logfiles[i].getPath(), conf);
+ try {
+ int count = 0;
+ for (; in.next(key, val); count++) {
+ byte [] tableName = key.getTablename();
+ byte [] regionName = key.getRegionName();
+ SequenceFile.Writer w = logWriters.get(regionName);
+ if (w == null) {
+ Path logfile = new Path(
+ HRegion.getRegionDir(
+ HTableDescriptor.getTableDir(rootDir, tableName),
+ HRegionInfo.encodeRegionName(regionName)),
+ HREGION_OLDLOGFILE_NAME);
+ Path oldlogfile = null;
+ SequenceFile.Reader old = null;
+ if (fs.exists(logfile)) {
+ LOG.warn("Old log file " + logfile +
+ " already exists. Copying existing file to new file");
+ oldlogfile = new Path(logfile.toString() + ".old");
+ fs.rename(logfile, oldlogfile);
+ old = new SequenceFile.Reader(fs, oldlogfile, conf);
+ }
+ w = SequenceFile.createWriter(fs, conf, logfile, HLogKey.class,
+ HLogEdit.class, getCompressionType(conf));
+ // Use copy of regionName; regionName object is reused inside in
+ // HStoreKey.getRegionName so its content changes as we iterate.
+ logWriters.put(regionName, w);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Creating new log file writer for path " + logfile +
+ " and region " + Bytes.toString(regionName));
+ }
+
+ if (old != null) {
+ // Copy from existing log file
+ HLogKey oldkey = new HLogKey();
+ HLogEdit oldval = new HLogEdit();
+ for (; old.next(oldkey, oldval); count++) {
+ if (LOG.isDebugEnabled() && count > 0 && count % 10000 == 0) {
+ LOG.debug("Copied " + count + " edits");
+ }
+ w.append(oldkey, oldval);
}
- w.append(oldkey, oldval);
+ old.close();
+ fs.delete(oldlogfile, true);
}
- old.close();
- fs.delete(oldlogfile, true);
}
+ w.append(key, val);
}
- w.append(key, val);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Applied " + count + " total edits from " +
- logfiles[i].getPath().toString());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Applied " + count + " total edits from " +
+ logfiles[i].getPath().toString());
+ }
+ } catch (IOException e) {
+ e = RemoteExceptionHandler.checkIOException(e);
+ if (!(e instanceof EOFException)) {
+ LOG.warn("Exception processing " + logfiles[i].getPath() +
+ " -- continuing. Possible DATA LOSS!", e);
+ }
+ } finally {
+ try {
+ in.close();
+ } catch (IOException e) {
+ LOG.warn("Close in finally threw exception -- continuing", e);
+ }
+ // Delete the input file now so we do not replay edits. We could
+ // have gotten here because of an exception. If so, probably
+ // nothing we can do about it. Replaying it, it could work but we
+ // could be stuck replaying for ever. Just continue though we
+ // could have lost some edits.
+ fs.delete(logfiles[i].getPath(), true);
}
} catch (IOException e) {
- e = RemoteExceptionHandler.checkIOException(e);
- if (!(e instanceof EOFException)) {
- LOG.warn("Exception processing " + logfiles[i].getPath() +
- " -- continuing. Possible DATA LOSS!", e);
- }
- } finally {
- try {
- in.close();
- } catch (IOException e) {
- LOG.warn("Close in finally threw exception -- continuing", e);
+ if (possiblyEmpty) {
+ continue;
}
- // Delete the input file now so we do not replay edits. We could
- // have gotten here because of an exception. If so, probably
- // nothing we can do about it. Replaying it, it could work but we
- // could be stuck replaying for ever. Just continue though we
- // could have lost some edits.
- fs.delete(logfiles[i].getPath(), true);
+ throw e;
}
}
} finally {