You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by cn...@apache.org on 2014/01/27 19:09:27 UTC
svn commit: r1561771 [1/3] - in
/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs: ./
dev-support/
src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/
src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bk...
Author: cnauroth
Date: Mon Jan 27 18:09:22 2014
New Revision: 1561771
URL: http://svn.apache.org/r1561771
Log:
Merge trunk to HDFS-4685.
Added:
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java
- copied unchanged from r1561769, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java
Modified:
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithQJM.apt.vm
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSUpgradeWithHA.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java
hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
Propchange: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs:r1556680-1561449
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1560768-1561769
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Jan 27 18:09:22 2014
@@ -148,6 +148,8 @@ Trunk (Unreleased)
HDFS-5721. sharedEditsImage in Namenode#initializeSharedEdits() should be
closed before method returns. (Ted Yu via junping_du)
+ HDFS-5138. Support HDFS upgrade in HA. (atm via todd)
+
OPTIMIZATIONS
BUG FIXES
@@ -594,6 +596,9 @@ Release 2.4.0 - UNRELEASED
HDFS-5806. balancer should set SoTimeout to avoid indefinite hangs.
(Nathan Roberts via Andrew Wang).
+ HDFS-5728. Block recovery will fail if the metafile does not have crc
+ for all chunks of the block (Vinay via kihwal)
+
BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS
HDFS-4985. Add storage type to the protocol and expose it in block report
@@ -1134,6 +1139,9 @@ Release 2.3.0 - UNRELEASED
HDFS-5789. Some of snapshot APIs missing checkOperation double check in fsn. (umamahesh)
+ HDFS-5343. When cat command is issued on snapshot files getting unexpected result.
+ (Sathish via umamahesh)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml Mon Jan 27 18:09:22 2014
@@ -361,5 +361,10 @@
<Class name="org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor" />
<Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.hdfs.DFSUtil"/>
+ <Method name="assertAllResultsEqual" />
+ <Bug pattern="NP_LOAD_OF_KNOWN_NULL_VALUE" />
+ </Match>
</FindBugsFilter>
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java Mon Jan 27 18:09:22 2014
@@ -18,6 +18,8 @@
package org.apache.hadoop.contrib.bkjournal;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.JournalManager;
import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream;
import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream;
@@ -660,6 +662,37 @@ public class BookKeeperJournalManager im
}
@Override
+ public void doPreUpgrade() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void doUpgrade(Storage storage) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getJournalCTime() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void doFinalize() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean canRollBack(StorageInfo storage, StorageInfo prevStorage,
+ int targetLayoutVersion) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void doRollback() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void close() throws IOException {
try {
bkc.close();
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java Mon Jan 27 18:09:22 2014
@@ -316,7 +316,7 @@ public class TestBookKeeperAsHASharedDir
} catch (IOException ioe) {
LOG.info("Got expected exception", ioe);
GenericTestUtils.assertExceptionContains(
- "Cannot start an HA namenode with name dirs that need recovery", ioe);
+ "storage directory does not exist or is not accessible", ioe);
}
}
Propchange: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1556680-1561449
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1560768-1561769
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java Mon Jan 27 18:09:22 2014
@@ -792,6 +792,9 @@ implements ByteBufferReadable, CanSetDro
currentNode = blockSeekTo(pos);
}
int realLen = (int) Math.min(len, (blockEnd - pos + 1L));
+ if (locatedBlocks.isLastBlockComplete()) {
+ realLen = (int) Math.min(realLen, locatedBlocks.getFileLength());
+ }
int result = readBuffer(strategy, off, realLen, corruptedBlockMap);
if (result >= 0) {
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Mon Jan 27 18:09:22 2014
@@ -43,6 +43,7 @@ import java.net.URISyntaxException;
import java.security.SecureRandom;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -574,10 +575,24 @@ public class DFSUtil {
}
return ret;
}
+
+ /**
+ * Get all of the RPC addresses of the individual NNs in a given nameservice.
+ *
+ * @param conf Configuration
+ * @param nsId the nameservice whose NNs addresses we want.
+ * @param defaultValue default address to return in case key is not found.
+ * @return A map from nnId -> RPC address of each NN in the nameservice.
+ */
+ public static Map<String, InetSocketAddress> getRpcAddressesForNameserviceId(
+ Configuration conf, String nsId, String defaultValue) {
+ return getAddressesForNameserviceId(conf, nsId, defaultValue,
+ DFS_NAMENODE_RPC_ADDRESS_KEY);
+ }
private static Map<String, InetSocketAddress> getAddressesForNameserviceId(
Configuration conf, String nsId, String defaultValue,
- String[] keys) {
+ String... keys) {
Collection<String> nnIds = getNameNodeIds(conf, nsId);
Map<String, InetSocketAddress> ret = Maps.newHashMap();
for (String nnId : emptyAsSingletonNull(nnIds)) {
@@ -1670,4 +1685,32 @@ public class DFSUtil {
}
return builder;
}
+
+ /**
+ * Assert that all objects in the collection are equal. Returns silently if
+ * so, throws an AssertionError if any object is not equal. All null values
+ * are considered equal.
+ *
+ * @param objects the collection of objects to check for equality.
+ */
+ public static void assertAllResultsEqual(Collection<?> objects) {
+ Object[] resultsArray = objects.toArray();
+
+ if (resultsArray.length == 0)
+ return;
+
+ for (int i = 0; i < resultsArray.length; i++) {
+ if (i == 0)
+ continue;
+ else {
+ Object currElement = resultsArray[i];
+ Object lastElement = resultsArray[i - 1];
+ if ((currElement == null && currElement != lastElement) ||
+ (currElement != null && !currElement.equals(lastElement))) {
+ throw new AssertionError("Not all elements match in results: " +
+ Arrays.toString(resultsArray));
+ }
+ }
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java Mon Jan 27 18:09:22 2014
@@ -26,22 +26,29 @@ import org.apache.hadoop.HadoopIllegalAr
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.List;
import java.util.Map;
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
@@ -305,4 +312,55 @@ public class HAUtil {
DFSClient dfsClient = dfs.getClient();
return RPC.getServerAddress(dfsClient.getNamenode());
}
+
+ /**
+ * Get an RPC proxy for each NN in an HA nameservice. Used when a given RPC
+ * call should be made on every NN in an HA nameservice, not just the active.
+ *
+ * @param conf configuration
+ * @param nsId the nameservice to get all of the proxies for.
+ * @return a list of RPC proxies for each NN in the nameservice.
+ * @throws IOException in the event of error.
+ */
+ public static List<ClientProtocol> getProxiesForAllNameNodesInNameservice(
+ Configuration conf, String nsId) throws IOException {
+ Map<String, InetSocketAddress> nnAddresses =
+ DFSUtil.getRpcAddressesForNameserviceId(conf, nsId, null);
+
+ List<ClientProtocol> namenodes = new ArrayList<ClientProtocol>();
+ for (InetSocketAddress nnAddress : nnAddresses.values()) {
+ NameNodeProxies.ProxyAndInfo<ClientProtocol> proxyInfo = null;
+ proxyInfo = NameNodeProxies.createNonHAProxy(conf,
+ nnAddress, ClientProtocol.class,
+ UserGroupInformation.getCurrentUser(), false);
+ namenodes.add(proxyInfo.getProxy());
+ }
+ return namenodes;
+ }
+
+ /**
+ * Used to ensure that at least one of the given HA NNs is currently in the
+ * active state..
+ *
+ * @param namenodes list of RPC proxies for each NN to check.
+ * @return true if at least one NN is active, false if all are in the standby state.
+ * @throws IOException in the event of error.
+ */
+ public static boolean isAtLeastOneActive(List<ClientProtocol> namenodes)
+ throws IOException {
+ for (ClientProtocol namenode : namenodes) {
+ try {
+ namenode.getFileInfo("/");
+ return true;
+ } catch (RemoteException re) {
+ IOException cause = re.unwrapRemoteException();
+ if (cause instanceof StandbyException) {
+ // This is expected to happen for a standby NN.
+ } else {
+ throw re;
+ }
+ }
+ }
+ return false;
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java Mon Jan 27 18:09:22 2014
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.qjournal.p
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto;
import org.apache.hadoop.hdfs.qjournal.protocol.RequestInfo;
+import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
@@ -151,4 +152,17 @@ interface AsyncLogger {
* StringBuilder. This is displayed on the NN web UI.
*/
public void appendReport(StringBuilder sb);
+
+ public ListenableFuture<Void> doPreUpgrade();
+
+ public ListenableFuture<Void> doUpgrade(StorageInfo sInfo);
+
+ public ListenableFuture<Void> doFinalize();
+
+ public ListenableFuture<Boolean> canRollBack(StorageInfo storage,
+ StorageInfo prevStorage, int targetLayoutVersion);
+
+ public ListenableFuture<Void> doRollback();
+
+ public ListenableFuture<Long> getJournalCTime();
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java Mon Jan 27 18:09:22 2014
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.qjournal.p
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto;
+import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
@@ -308,4 +309,71 @@ class AsyncLoggerSet {
}
return QuorumCall.create(calls);
}
+
+ QuorumCall<AsyncLogger, Void> doPreUpgrade() {
+ Map<AsyncLogger, ListenableFuture<Void>> calls =
+ Maps.newHashMap();
+ for (AsyncLogger logger : loggers) {
+ ListenableFuture<Void> future =
+ logger.doPreUpgrade();
+ calls.put(logger, future);
+ }
+ return QuorumCall.create(calls);
+ }
+
+ public QuorumCall<AsyncLogger, Void> doUpgrade(StorageInfo sInfo) {
+ Map<AsyncLogger, ListenableFuture<Void>> calls =
+ Maps.newHashMap();
+ for (AsyncLogger logger : loggers) {
+ ListenableFuture<Void> future =
+ logger.doUpgrade(sInfo);
+ calls.put(logger, future);
+ }
+ return QuorumCall.create(calls);
+ }
+
+ public QuorumCall<AsyncLogger, Void> doFinalize() {
+ Map<AsyncLogger, ListenableFuture<Void>> calls =
+ Maps.newHashMap();
+ for (AsyncLogger logger : loggers) {
+ ListenableFuture<Void> future =
+ logger.doFinalize();
+ calls.put(logger, future);
+ }
+ return QuorumCall.create(calls);
+ }
+
+ public QuorumCall<AsyncLogger, Boolean> canRollBack(StorageInfo storage,
+ StorageInfo prevStorage, int targetLayoutVersion) {
+ Map<AsyncLogger, ListenableFuture<Boolean>> calls =
+ Maps.newHashMap();
+ for (AsyncLogger logger : loggers) {
+ ListenableFuture<Boolean> future =
+ logger.canRollBack(storage, prevStorage, targetLayoutVersion);
+ calls.put(logger, future);
+ }
+ return QuorumCall.create(calls);
+ }
+
+ public QuorumCall<AsyncLogger, Void> doRollback() {
+ Map<AsyncLogger, ListenableFuture<Void>> calls =
+ Maps.newHashMap();
+ for (AsyncLogger logger : loggers) {
+ ListenableFuture<Void> future =
+ logger.doRollback();
+ calls.put(logger, future);
+ }
+ return QuorumCall.create(calls);
+ }
+
+ public QuorumCall<AsyncLogger, Long> getJournalCTime() {
+ Map<AsyncLogger, ListenableFuture<Long>> calls =
+ Maps.newHashMap();
+ for (AsyncLogger logger : loggers) {
+ ListenableFuture<Long> future = logger.getJournalCTime();
+ calls.put(logger, future);
+ }
+ return QuorumCall.create(calls);
+ }
+
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java Mon Jan 27 18:09:22 2014
@@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.qjournal.p
import org.apache.hadoop.hdfs.qjournal.protocolPB.QJournalProtocolPB;
import org.apache.hadoop.hdfs.qjournal.protocolPB.QJournalProtocolTranslatorPB;
import org.apache.hadoop.hdfs.qjournal.server.GetJournalEditServlet;
+import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
@@ -564,6 +565,72 @@ public class IPCLoggerChannel implements
}
});
}
+
+ @Override
+ public ListenableFuture<Void> doPreUpgrade() {
+ return executor.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws IOException {
+ getProxy().doPreUpgrade(journalId);
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public ListenableFuture<Void> doUpgrade(final StorageInfo sInfo) {
+ return executor.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws IOException {
+ getProxy().doUpgrade(journalId, sInfo);
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public ListenableFuture<Void> doFinalize() {
+ return executor.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws IOException {
+ getProxy().doFinalize(journalId);
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public ListenableFuture<Boolean> canRollBack(final StorageInfo storage,
+ final StorageInfo prevStorage, final int targetLayoutVersion) {
+ return executor.submit(new Callable<Boolean>() {
+ @Override
+ public Boolean call() throws IOException {
+ return getProxy().canRollBack(journalId, storage, prevStorage,
+ targetLayoutVersion);
+ }
+ });
+ }
+
+ @Override
+ public ListenableFuture<Void> doRollback() {
+ return executor.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws IOException {
+ getProxy().doRollback(journalId);
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public ListenableFuture<Long> getJournalCTime() {
+ return executor.submit(new Callable<Long>() {
+ @Override
+ public Long call() throws IOException {
+ return getProxy().getJournalCTime(journalId);
+ }
+ });
+ }
@Override
public String toString() {
@@ -636,4 +703,5 @@ public class IPCLoggerChannel implements
private boolean hasHttpServerEndPoint() {
return httpServerURL != null;
}
+
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java Mon Jan 27 18:09:22 2014
@@ -34,10 +34,13 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournalStateResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto;
+import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream;
import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream;
import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream;
@@ -77,8 +80,14 @@ public class QuorumJournalManager implem
// Since these don't occur during normal operation, we can
// use rather lengthy timeouts, and don't need to make them
// configurable.
- private static final int FORMAT_TIMEOUT_MS = 60000;
- private static final int HASDATA_TIMEOUT_MS = 60000;
+ private static final int FORMAT_TIMEOUT_MS = 60000;
+ private static final int HASDATA_TIMEOUT_MS = 60000;
+ private static final int CAN_ROLL_BACK_TIMEOUT_MS = 60000;
+ private static final int FINALIZE_TIMEOUT_MS = 60000;
+ private static final int PRE_UPGRADE_TIMEOUT_MS = 60000;
+ private static final int ROLL_BACK_TIMEOUT_MS = 60000;
+ private static final int UPGRADE_TIMEOUT_MS = 60000;
+ private static final int GET_JOURNAL_CTIME_TIMEOUT_MS = 60000;
private final Configuration conf;
private final URI uri;
@@ -492,4 +501,131 @@ public class QuorumJournalManager implem
return loggers;
}
+ @Override
+ public void doPreUpgrade() throws IOException {
+ QuorumCall<AsyncLogger, Void> call = loggers.doPreUpgrade();
+ try {
+ call.waitFor(loggers.size(), loggers.size(), 0, PRE_UPGRADE_TIMEOUT_MS,
+ "doPreUpgrade");
+
+ if (call.countExceptions() > 0) {
+ call.rethrowException("Could not do pre-upgrade of one or more JournalNodes");
+ }
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted waiting for doPreUpgrade() response");
+ } catch (TimeoutException e) {
+ throw new IOException("Timed out waiting for doPreUpgrade() response");
+ }
+ }
+
+ @Override
+ public void doUpgrade(Storage storage) throws IOException {
+ QuorumCall<AsyncLogger, Void> call = loggers.doUpgrade(storage);
+ try {
+ call.waitFor(loggers.size(), loggers.size(), 0, UPGRADE_TIMEOUT_MS,
+ "doUpgrade");
+
+ if (call.countExceptions() > 0) {
+ call.rethrowException("Could not perform upgrade of one or more JournalNodes");
+ }
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted waiting for doUpgrade() response");
+ } catch (TimeoutException e) {
+ throw new IOException("Timed out waiting for doUpgrade() response");
+ }
+ }
+
+ @Override
+ public void doFinalize() throws IOException {
+ QuorumCall<AsyncLogger, Void> call = loggers.doFinalize();
+ try {
+ call.waitFor(loggers.size(), loggers.size(), 0, FINALIZE_TIMEOUT_MS,
+ "doFinalize");
+
+ if (call.countExceptions() > 0) {
+ call.rethrowException("Could not finalize one or more JournalNodes");
+ }
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted waiting for doFinalize() response");
+ } catch (TimeoutException e) {
+ throw new IOException("Timed out waiting for doFinalize() response");
+ }
+ }
+
+ @Override
+ public boolean canRollBack(StorageInfo storage, StorageInfo prevStorage,
+ int targetLayoutVersion) throws IOException {
+ QuorumCall<AsyncLogger, Boolean> call = loggers.canRollBack(storage,
+ prevStorage, targetLayoutVersion);
+ try {
+ call.waitFor(loggers.size(), loggers.size(), 0, CAN_ROLL_BACK_TIMEOUT_MS,
+ "lockSharedStorage");
+
+ if (call.countExceptions() > 0) {
+ call.rethrowException("Could not check if roll back possible for"
+ + " one or more JournalNodes");
+ }
+
+ // Either they all return the same thing or this call fails, so we can
+ // just return the first result.
+ DFSUtil.assertAllResultsEqual(call.getResults().values());
+ for (Boolean result : call.getResults().values()) {
+ return result;
+ }
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted waiting for lockSharedStorage() " +
+ "response");
+ } catch (TimeoutException e) {
+ throw new IOException("Timed out waiting for lockSharedStorage() " +
+ "response");
+ }
+
+ throw new AssertionError("Unreachable code.");
+ }
+
+ @Override
+ public void doRollback() throws IOException {
+ QuorumCall<AsyncLogger, Void> call = loggers.doRollback();
+ try {
+ call.waitFor(loggers.size(), loggers.size(), 0, ROLL_BACK_TIMEOUT_MS,
+ "doRollback");
+
+ if (call.countExceptions() > 0) {
+ call.rethrowException("Could not perform rollback of one or more JournalNodes");
+ }
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted waiting for doFinalize() response");
+ } catch (TimeoutException e) {
+ throw new IOException("Timed out waiting for doFinalize() response");
+ }
+ }
+
+ @Override
+ public long getJournalCTime() throws IOException {
+ QuorumCall<AsyncLogger, Long> call = loggers.getJournalCTime();
+ try {
+ call.waitFor(loggers.size(), loggers.size(), 0,
+ GET_JOURNAL_CTIME_TIMEOUT_MS, "getJournalCTime");
+
+ if (call.countExceptions() > 0) {
+ call.rethrowException("Could not journal CTime for one "
+ + "more JournalNodes");
+ }
+
+ // Either they all return the same thing or this call fails, so we can
+ // just return the first result.
+ DFSUtil.assertAllResultsEqual(call.getResults().values());
+ for (Long result : call.getResults().values()) {
+ return result;
+ }
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted waiting for getJournalCTime() " +
+ "response");
+ } catch (TimeoutException e) {
+ throw new IOException("Timed out waiting for getJournalCTime() " +
+ "response");
+ }
+
+ throw new AssertionError("Unreachable code.");
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java Mon Jan 27 18:09:22 2014
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.qjournal.p
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto;
import org.apache.hadoop.hdfs.qjournal.server.JournalNode;
+import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.JournalManager;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.security.KerberosInfo;
@@ -143,4 +144,17 @@ public interface QJournalProtocol {
*/
public void acceptRecovery(RequestInfo reqInfo,
SegmentStateProto stateToAccept, URL fromUrl) throws IOException;
+
+ public void doPreUpgrade(String journalId) throws IOException;
+
+ public void doUpgrade(String journalId, StorageInfo sInfo) throws IOException;
+
+ public void doFinalize(String journalId) throws IOException;
+
+ public Boolean canRollBack(String journalId, StorageInfo storage,
+ StorageInfo prevStorage, int targetLayoutVersion) throws IOException;
+
+ public void doRollback(String journalId) throws IOException;
+
+ public Long getJournalCTime(String journalId) throws IOException;
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java Mon Jan 27 18:09:22 2014
@@ -17,17 +17,35 @@
*/
package org.apache.hadoop.hdfs.qjournal.protocolPB;
+import java.io.IOException;
+import java.net.URL;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocolPB.JournalProtocolPB;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocol;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.AcceptRecoveryRequestProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.AcceptRecoveryResponseProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.CanRollBackRequestProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.CanRollBackResponseProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.DoFinalizeRequestProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.DoFinalizeResponseProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.DoPreUpgradeRequestProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.DoPreUpgradeResponseProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.DoRollbackRequestProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.DoRollbackResponseProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.DoUpgradeRequestProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.DoUpgradeResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.FinalizeLogSegmentRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.FinalizeLogSegmentResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.FormatRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.FormatResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetEditLogManifestRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetEditLogManifestResponseProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournalCTimeRequestProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournalCTimeResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournalStateRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournalStateResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.HeartbeatRequestProto;
@@ -39,8 +57,6 @@ import org.apache.hadoop.hdfs.qjournal.p
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.JournalResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochResponseProto;
-import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.AcceptRecoveryRequestProto;
-import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.AcceptRecoveryResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PurgeLogsRequestProto;
@@ -48,13 +64,11 @@ import org.apache.hadoop.hdfs.qjournal.p
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.StartLogSegmentRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.StartLogSegmentResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.RequestInfo;
+import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
-import java.io.IOException;
-import java.net.URL;
-
/**
* Implementation for protobuf service that forwards requests
* received on {@link JournalProtocolPB} to the
@@ -244,4 +258,79 @@ public class QJournalProtocolServerSideT
reqInfo.hasCommittedTxId() ?
reqInfo.getCommittedTxId() : HdfsConstants.INVALID_TXID);
}
+
+
+ @Override
+ public DoPreUpgradeResponseProto doPreUpgrade(RpcController controller,
+ DoPreUpgradeRequestProto request) throws ServiceException {
+ try {
+ impl.doPreUpgrade(convert(request.getJid()));
+ return DoPreUpgradeResponseProto.getDefaultInstance();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public DoUpgradeResponseProto doUpgrade(RpcController controller,
+ DoUpgradeRequestProto request) throws ServiceException {
+ try {
+ impl.doUpgrade(convert(request.getJid()),
+ PBHelper.convert(request.getSInfo()));
+ return DoUpgradeResponseProto.getDefaultInstance();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public DoFinalizeResponseProto doFinalize(RpcController controller,
+ DoFinalizeRequestProto request) throws ServiceException {
+ try {
+ impl.doFinalize(convert(request.getJid()));
+ return DoFinalizeResponseProto.getDefaultInstance();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public CanRollBackResponseProto canRollBack(RpcController controller,
+ CanRollBackRequestProto request) throws ServiceException {
+ try {
+ Boolean result = impl.canRollBack(convert(request.getJid()),
+ PBHelper.convert(request.getStorage()),
+ PBHelper.convert(request.getPrevStorage()),
+ request.getTargetLayoutVersion());
+ return CanRollBackResponseProto.newBuilder()
+ .setCanRollBack(result)
+ .build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public DoRollbackResponseProto doRollback(RpcController controller, DoRollbackRequestProto request)
+ throws ServiceException {
+ try {
+ impl.doRollback(convert(request.getJid()));
+ return DoRollbackResponseProto.getDefaultInstance();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public GetJournalCTimeResponseProto getJournalCTime(RpcController controller,
+ GetJournalCTimeRequestProto request) throws ServiceException {
+ try {
+ Long resultCTime = impl.getJournalCTime(convert(request.getJid()));
+ return GetJournalCTimeResponseProto.newBuilder()
+ .setResultCTime(resultCTime)
+ .build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java Mon Jan 27 18:09:22 2014
@@ -23,13 +23,23 @@ import java.net.URL;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.protocolPB.JournalProtocolPB;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocol;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.AcceptRecoveryRequestProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.CanRollBackRequestProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.CanRollBackResponseProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.DoFinalizeRequestProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.DoPreUpgradeRequestProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.DoRollbackRequestProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.DoUpgradeRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.FinalizeLogSegmentRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.FormatRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetEditLogManifestRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetEditLogManifestResponseProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournalCTimeRequestProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournalCTimeResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournalStateRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournalStateResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.HeartbeatRequestProto;
@@ -39,7 +49,6 @@ import org.apache.hadoop.hdfs.qjournal.p
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.JournalRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochResponseProto;
-import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.AcceptRecoveryRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PurgeLogsRequestProto;
@@ -47,6 +56,7 @@ import org.apache.hadoop.hdfs.qjournal.p
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.StartLogSegmentRequestProto;
import org.apache.hadoop.hdfs.qjournal.protocol.RequestInfo;
+import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.ipc.ProtobufHelper;
@@ -277,4 +287,85 @@ public class QJournalProtocolTranslatorP
RPC.getProtocolVersion(QJournalProtocolPB.class), methodName);
}
+ @Override
+ public void doPreUpgrade(String jid) throws IOException {
+ try {
+ rpcProxy.doPreUpgrade(NULL_CONTROLLER,
+ DoPreUpgradeRequestProto.newBuilder()
+ .setJid(convertJournalId(jid))
+ .build());
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public void doUpgrade(String journalId, StorageInfo sInfo) throws IOException {
+ try {
+ rpcProxy.doUpgrade(NULL_CONTROLLER,
+ DoUpgradeRequestProto.newBuilder()
+ .setJid(convertJournalId(journalId))
+ .setSInfo(PBHelper.convert(sInfo))
+ .build());
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public void doFinalize(String jid) throws IOException {
+ try {
+ rpcProxy.doFinalize(NULL_CONTROLLER,
+ DoFinalizeRequestProto.newBuilder()
+ .setJid(convertJournalId(jid))
+ .build());
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public Boolean canRollBack(String journalId, StorageInfo storage,
+ StorageInfo prevStorage, int targetLayoutVersion) throws IOException {
+ try {
+ CanRollBackResponseProto response = rpcProxy.canRollBack(
+ NULL_CONTROLLER,
+ CanRollBackRequestProto.newBuilder()
+ .setJid(convertJournalId(journalId))
+ .setStorage(PBHelper.convert(storage))
+ .setPrevStorage(PBHelper.convert(prevStorage))
+ .setTargetLayoutVersion(targetLayoutVersion)
+ .build());
+ return response.getCanRollBack();
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public void doRollback(String journalId) throws IOException {
+ try {
+ rpcProxy.doRollback(NULL_CONTROLLER,
+ DoRollbackRequestProto.newBuilder()
+ .setJid(convertJournalId(journalId))
+ .build());
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public Long getJournalCTime(String journalId) throws IOException {
+ try {
+ GetJournalCTimeResponseProto response = rpcProxy.getJournalCTime(
+ NULL_CONTROLLER,
+ GetJournalCTimeRequestProto.newBuilder()
+ .setJid(convertJournalId(journalId))
+ .build());
+ return response.getResultCTime();
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java Mon Jan 27 18:09:22 2014
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager;
import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.FileJournalManager;
import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile;
import org.apache.hadoop.hdfs.server.namenode.GetImageServlet;
@@ -139,20 +140,26 @@ public class GetJournalEditServlet exten
private boolean checkStorageInfoOrSendError(JNStorage storage,
HttpServletRequest request, HttpServletResponse response)
throws IOException {
- String myStorageInfoString = storage.toColonSeparatedString();
+ int myNsId = storage.getNamespaceID();
+ String myClusterId = storage.getClusterID();
+
String theirStorageInfoString = StringEscapeUtils.escapeHtml(
request.getParameter(STORAGEINFO_PARAM));
- if (theirStorageInfoString != null
- && !myStorageInfoString.equals(theirStorageInfoString)) {
- String msg = "This node has storage info '" + myStorageInfoString
- + "' but the requesting node expected '"
- + theirStorageInfoString + "'";
-
- response.sendError(HttpServletResponse.SC_FORBIDDEN, msg);
- LOG.warn("Received an invalid request file transfer request from " +
- request.getRemoteAddr() + ": " + msg);
- return false;
+ if (theirStorageInfoString != null) {
+ int theirNsId = StorageInfo.getNsIdFromColonSeparatedString(
+ theirStorageInfoString);
+ String theirClusterId = StorageInfo.getClusterIdFromColonSeparatedString(
+ theirStorageInfoString);
+ if (myNsId != theirNsId || !myClusterId.equals(theirClusterId)) {
+ String msg = "This node has namespaceId '" + myNsId + " and clusterId '"
+ + myClusterId + "' but the requesting node expected '" + theirNsId
+ + "' and '" + theirClusterId + "'";
+ response.sendError(HttpServletResponse.SC_FORBIDDEN, msg);
+ LOG.warn("Received an invalid request file transfer request from " +
+ request.getRemoteAddr() + ": " + msg);
+ return false;
+ }
}
return true;
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java Mon Jan 27 18:09:22 2014
@@ -130,6 +130,10 @@ class JNStorage extends Storage {
return new File(sd.getCurrentDir(), "paxos");
}
+ File getRoot() {
+ return sd.getRoot();
+ }
+
/**
* Remove any log files and associated paxos files which are older than
* the given txid.
@@ -182,12 +186,15 @@ class JNStorage extends Storage {
unlockAll();
sd.clearDirectory();
writeProperties(sd);
+ createPaxosDir();
+ analyzeStorage();
+ }
+
+ void createPaxosDir() throws IOException {
if (!getPaxosDir().mkdirs()) {
throw new IOException("Could not create paxos dir: " + getPaxosDir());
}
- analyzeStorage();
}
-
void analyzeStorage() throws IOException {
this.state = sd.analyzeStorage(StartupOption.REGULAR, this);
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java Mon Jan 27 18:09:22 2014
@@ -37,12 +37,14 @@ import org.apache.hadoop.hdfs.protocol.H
import org.apache.hadoop.hdfs.qjournal.protocol.JournalNotFormattedException;
import org.apache.hadoop.hdfs.qjournal.protocol.JournalOutOfSyncException;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocol;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PersistedRecoveryPaxosData;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto;
import org.apache.hadoop.hdfs.qjournal.protocol.RequestInfo;
import org.apache.hadoop.hdfs.server.common.StorageErrorReporter;
+import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream;
import org.apache.hadoop.hdfs.server.namenode.FileJournalManager;
import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile;
@@ -73,7 +75,7 @@ import com.google.protobuf.TextFormat;
* Each such journal is entirely independent despite being hosted by
* the same JVM.
*/
-class Journal implements Closeable {
+public class Journal implements Closeable {
static final Log LOG = LogFactory.getLog(Journal.class);
@@ -122,8 +124,8 @@ class Journal implements Closeable {
*/
private BestEffortLongFile committedTxnId;
- private static final String LAST_PROMISED_FILENAME = "last-promised-epoch";
- private static final String LAST_WRITER_EPOCH = "last-writer-epoch";
+ public static final String LAST_PROMISED_FILENAME = "last-promised-epoch";
+ public static final String LAST_WRITER_EPOCH = "last-writer-epoch";
private static final String COMMITTED_TXID_FILENAME = "committed-txid";
private final FileJournalManager fjm;
@@ -627,7 +629,7 @@ class Journal implements Closeable {
}
/**
- * @see QJournalProtocol#getEditLogManifest(String, long)
+ * @see QJournalProtocol#getEditLogManifest(String, long, boolean)
*/
public RemoteEditLogManifest getEditLogManifest(long sinceTxId,
boolean inProgressOk) throws IOException {
@@ -728,7 +730,7 @@ class Journal implements Closeable {
}
/**
- * @see QJournalProtocol#acceptRecovery(RequestInfo, SegmentStateProto, URL)
+ * @see QJournalProtocol#acceptRecovery(RequestInfo, QJournalProtocolProtos.SegmentStateProto, URL)
*/
public synchronized void acceptRecovery(RequestInfo reqInfo,
SegmentStateProto segment, URL fromUrl)
@@ -980,4 +982,62 @@ class Journal implements Closeable {
}
}
}
+
+ public synchronized void doPreUpgrade() throws IOException {
+ storage.getJournalManager().doPreUpgrade();
+ }
+
+ public synchronized void doUpgrade(StorageInfo sInfo) throws IOException {
+ long oldCTime = storage.getCTime();
+ storage.cTime = sInfo.cTime;
+ int oldLV = storage.getLayoutVersion();
+ storage.layoutVersion = sInfo.layoutVersion;
+ LOG.info("Starting upgrade of edits directory: "
+ + ".\n old LV = " + oldLV
+ + "; old CTime = " + oldCTime
+ + ".\n new LV = " + storage.getLayoutVersion()
+ + "; new CTime = " + storage.getCTime());
+ storage.getJournalManager().doUpgrade(storage);
+ storage.createPaxosDir();
+
+ // Copy over the contents of the epoch data files to the new dir.
+ File currentDir = storage.getSingularStorageDir().getCurrentDir();
+ File previousDir = storage.getSingularStorageDir().getPreviousDir();
+
+ PersistentLongFile prevLastPromisedEpoch = new PersistentLongFile(
+ new File(previousDir, LAST_PROMISED_FILENAME), 0);
+ PersistentLongFile prevLastWriterEpoch = new PersistentLongFile(
+ new File(previousDir, LAST_WRITER_EPOCH), 0);
+
+ lastPromisedEpoch = new PersistentLongFile(
+ new File(currentDir, LAST_PROMISED_FILENAME), 0);
+ lastWriterEpoch = new PersistentLongFile(
+ new File(currentDir, LAST_WRITER_EPOCH), 0);
+
+ lastPromisedEpoch.set(prevLastPromisedEpoch.get());
+ lastWriterEpoch.set(prevLastWriterEpoch.get());
+ }
+
+ public synchronized void doFinalize() throws IOException {
+ LOG.info("Finalizing upgrade for journal "
+ + storage.getRoot() + "."
+ + (storage.getLayoutVersion()==0 ? "" :
+ "\n cur LV = " + storage.getLayoutVersion()
+ + "; cur CTime = " + storage.getCTime()));
+ storage.getJournalManager().doFinalize();
+ }
+
+ public Boolean canRollBack(StorageInfo storage, StorageInfo prevStorage,
+ int targetLayoutVersion) throws IOException {
+ return this.storage.getJournalManager().canRollBack(storage, prevStorage,
+ targetLayoutVersion);
+ }
+
+ public void doRollback() throws IOException {
+ storage.getJournalManager().doRollback();
+ }
+
+ public Long getJournalCTime() throws IOException {
+ return storage.getJournalManager().getJournalCTime();
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java Mon Jan 27 18:09:22 2014
@@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager;
import org.apache.hadoop.hdfs.server.common.StorageErrorReporter;
+import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
@@ -285,4 +286,31 @@ public class JournalNode implements Tool
StringUtils.startupShutdownMessage(JournalNode.class, args, LOG);
System.exit(ToolRunner.run(new JournalNode(), args));
}
+
+ public void doPreUpgrade(String journalId) throws IOException {
+ getOrCreateJournal(journalId).doPreUpgrade();
+ }
+
+ public void doUpgrade(String journalId, StorageInfo sInfo) throws IOException {
+ getOrCreateJournal(journalId).doUpgrade(sInfo);
+ }
+
+ public void doFinalize(String journalId) throws IOException {
+ getOrCreateJournal(journalId).doFinalize();
+ }
+
+ public Boolean canRollBack(String journalId, StorageInfo storage,
+ StorageInfo prevStorage, int targetLayoutVersion) throws IOException {
+ return getOrCreateJournal(journalId).canRollBack(storage, prevStorage,
+ targetLayoutVersion);
+ }
+
+ public void doRollback(String journalId) throws IOException {
+ getOrCreateJournal(journalId).doRollback();
+ }
+
+ public Long getJournalCTime(String journalId) throws IOException {
+ return getOrCreateJournal(journalId).getJournalCTime();
+ }
+
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java Mon Jan 27 18:09:22 2014
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.qjournal.p
import org.apache.hadoop.hdfs.qjournal.protocol.RequestInfo;
import org.apache.hadoop.hdfs.qjournal.protocolPB.QJournalProtocolPB;
import org.apache.hadoop.hdfs.qjournal.protocolPB.QJournalProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
@@ -205,4 +206,35 @@ class JournalNodeRpcServer implements QJ
.acceptRecovery(reqInfo, log, fromUrl);
}
+ @Override
+ public void doPreUpgrade(String journalId) throws IOException {
+ jn.doPreUpgrade(journalId);
+ }
+
+ @Override
+ public void doUpgrade(String journalId, StorageInfo sInfo) throws IOException {
+ jn.doUpgrade(journalId, sInfo);
+ }
+
+ @Override
+ public void doFinalize(String journalId) throws IOException {
+ jn.doFinalize(journalId);
+ }
+
+ @Override
+ public Boolean canRollBack(String journalId, StorageInfo storage,
+ StorageInfo prevStorage, int targetLayoutVersion)
+ throws IOException {
+ return jn.canRollBack(journalId, storage, prevStorage, targetLayoutVersion);
+ }
+
+ @Override
+ public void doRollback(String journalId) throws IOException {
+ jn.doRollback(journalId);
+ }
+
+ @Override
+ public Long getJournalCTime(String journalId) throws IOException {
+ return jn.getJournalCTime(journalId);
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java Mon Jan 27 18:09:22 2014
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.server.common;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
@@ -26,26 +25,23 @@ import java.lang.management.ManagementFa
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
import java.util.ArrayList;
-import java.util.List;
import java.util.Iterator;
+import java.util.List;
import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.LayoutVersion;
-import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
-import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.util.VersionInfo;
-import com.google.common.base.Preconditions;
-
import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
@@ -82,7 +78,6 @@ public abstract class Storage extends St
public static final int[] LAYOUT_VERSIONS_203 = {-19, -31};
public static final String STORAGE_FILE_LOCK = "in_use.lock";
- protected static final String STORAGE_FILE_VERSION = "VERSION";
public static final String STORAGE_DIR_CURRENT = "current";
public static final String STORAGE_DIR_PREVIOUS = "previous";
public static final String STORAGE_TMP_REMOVED = "removed.tmp";
@@ -126,22 +121,24 @@ public abstract class Storage extends St
private class DirIterator implements Iterator<StorageDirectory> {
StorageDirType dirType;
+ boolean includeShared;
int prevIndex; // for remove()
int nextIndex; // for next()
- DirIterator(StorageDirType dirType) {
+ DirIterator(StorageDirType dirType, boolean includeShared) {
this.dirType = dirType;
this.nextIndex = 0;
this.prevIndex = 0;
+ this.includeShared = includeShared;
}
@Override
public boolean hasNext() {
if (storageDirs.isEmpty() || nextIndex >= storageDirs.size())
return false;
- if (dirType != null) {
+ if (dirType != null || !includeShared) {
while (nextIndex < storageDirs.size()) {
- if (getStorageDir(nextIndex).getStorageDirType().isOfType(dirType))
+ if (shouldReturnNextDir())
break;
nextIndex++;
}
@@ -156,9 +153,9 @@ public abstract class Storage extends St
StorageDirectory sd = getStorageDir(nextIndex);
prevIndex = nextIndex;
nextIndex++;
- if (dirType != null) {
+ if (dirType != null || !includeShared) {
while (nextIndex < storageDirs.size()) {
- if (getStorageDir(nextIndex).getStorageDirType().isOfType(dirType))
+ if (shouldReturnNextDir())
break;
nextIndex++;
}
@@ -172,6 +169,12 @@ public abstract class Storage extends St
storageDirs.remove(prevIndex); // remove last returned element
hasNext(); // reset nextIndex to correct place
}
+
+ private boolean shouldReturnNextDir() {
+ StorageDirectory sd = getStorageDir(nextIndex);
+ return (dirType == null || sd.getStorageDirType().isOfType(dirType)) &&
+ (includeShared || !sd.isShared());
+ }
}
/**
@@ -203,7 +206,27 @@ public abstract class Storage extends St
* them via the Iterator
*/
public Iterator<StorageDirectory> dirIterator(StorageDirType dirType) {
- return new DirIterator(dirType);
+ return dirIterator(dirType, true);
+ }
+
+ /**
+ * Return all entries in storageDirs, potentially excluding shared dirs.
+ * @param includeShared whether or not to include shared dirs.
+ * @return an iterator over the configured storage dirs.
+ */
+ public Iterator<StorageDirectory> dirIterator(boolean includeShared) {
+ return dirIterator(null, includeShared);
+ }
+
+ /**
+ * @param dirType all entries will be of this type of dir
+ * @param includeShared true to include any shared directories,
+ * false otherwise
+ * @return an iterator over the configured storage dirs.
+ */
+ public Iterator<StorageDirectory> dirIterator(StorageDirType dirType,
+ boolean includeShared) {
+ return new DirIterator(dirType, includeShared);
}
public Iterable<StorageDirectory> dirIterable(final StorageDirType dirType) {
@@ -233,7 +256,9 @@ public abstract class Storage extends St
@InterfaceAudience.Private
public static class StorageDirectory implements FormatConfirmable {
final File root; // root directory
- final boolean useLock; // flag to enable storage lock
+ // whether or not this dir is shared between two separate NNs for HA, or
+ // between multiple block pools in the case of federation.
+ final boolean isShared;
final StorageDirType dirType; // storage dir type
FileLock lock; // storage lock
@@ -241,11 +266,11 @@ public abstract class Storage extends St
public StorageDirectory(File dir) {
// default dirType is null
- this(dir, null, true);
+ this(dir, null, false);
}
public StorageDirectory(File dir, StorageDirType dirType) {
- this(dir, dirType, true);
+ this(dir, dirType, false);
}
public void setStorageUuid(String storageUuid) {
@@ -260,14 +285,14 @@ public abstract class Storage extends St
* Constructor
* @param dir directory corresponding to the storage
* @param dirType storage directory type
- * @param useLock true - enables locking on the storage directory and false
- * disables locking
+ * @param isShared whether or not this dir is shared between two NNs. true
+ * disables locking on the storage directory, false enables locking
*/
- public StorageDirectory(File dir, StorageDirType dirType, boolean useLock) {
+ public StorageDirectory(File dir, StorageDirType dirType, boolean isShared) {
this.root = dir;
this.lock = null;
this.dirType = dirType;
- this.useLock = useLock;
+ this.isShared = isShared;
}
/**
@@ -621,6 +646,10 @@ public abstract class Storage extends St
return true;
}
+
+ public boolean isShared() {
+ return isShared;
+ }
/**
@@ -635,7 +664,7 @@ public abstract class Storage extends St
* @throws IOException if locking fails
*/
public void lock() throws IOException {
- if (!useLock) {
+ if (isShared()) {
LOG.info("Locking is disabled");
return;
}
@@ -890,22 +919,6 @@ public abstract class Storage extends St
}
/**
- * Get common storage fields.
- * Should be overloaded if additional fields need to be get.
- *
- * @param props
- * @throws IOException
- */
- protected void setFieldsFromProperties(
- Properties props, StorageDirectory sd) throws IOException {
- setLayoutVersion(props, sd);
- setNamespaceID(props, sd);
- setStorageType(props, sd);
- setcTime(props, sd);
- setClusterId(props, layoutVersion, sd);
- }
-
- /**
* Set common storage fields into the given properties object.
* Should be overloaded if additional fields need to be set.
*
@@ -923,22 +936,29 @@ public abstract class Storage extends St
}
props.setProperty("cTime", String.valueOf(cTime));
}
-
+
/**
- * Read properties from the VERSION file in the given storage directory.
+ * Get common storage fields.
+ * Should be overloaded if additional fields need to be get.
+ *
+ * @param props
+ * @throws IOException
*/
- public void readProperties(StorageDirectory sd) throws IOException {
- Properties props = readPropertiesFile(sd.getVersionFile());
- setFieldsFromProperties(props, sd);
+ protected void setFieldsFromProperties(
+ Properties props, StorageDirectory sd) throws IOException {
+ super.setFieldsFromProperties(props, sd);
+ setStorageType(props, sd);
}
-
- /**
- * Read properties from the the previous/VERSION file in the given storage directory.
- */
- public void readPreviousVersionProperties(StorageDirectory sd)
- throws IOException {
- Properties props = readPropertiesFile(sd.getPreviousVersionFile());
- setFieldsFromProperties(props, sd);
+
+ /** Validate and set storage type from {@link Properties}*/
+ protected void setStorageType(Properties props, StorageDirectory sd)
+ throws InconsistentFSStateException {
+ NodeType type = NodeType.valueOf(getProperty(props, sd, "storageType"));
+ if (!storageType.equals(type)) {
+ throw new InconsistentFSStateException(sd.root,
+ "node type is incompatible with others.");
+ }
+ storageType = type;
}
/**
@@ -947,10 +967,15 @@ public abstract class Storage extends St
public void writeProperties(StorageDirectory sd) throws IOException {
writeProperties(sd.getVersionFile(), sd);
}
-
+
public void writeProperties(File to, StorageDirectory sd) throws IOException {
Properties props = new Properties();
setPropertiesFromFields(props, sd);
+ writeProperties(to, sd, props);
+ }
+
+ public static void writeProperties(File to, StorageDirectory sd,
+ Properties props) throws IOException {
RandomAccessFile file = new RandomAccessFile(to, "rws");
FileOutputStream out = null;
try {
@@ -977,23 +1002,6 @@ public abstract class Storage extends St
file.close();
}
}
-
- public static Properties readPropertiesFile(File from) throws IOException {
- RandomAccessFile file = new RandomAccessFile(from, "rws");
- FileInputStream in = null;
- Properties props = new Properties();
- try {
- in = new FileInputStream(file.getFD());
- file.seek(0);
- props.load(in);
- } finally {
- if (in != null) {
- in.close();
- }
- file.close();
- }
- return props;
- }
public static void rename(File from, File to) throws IOException {
if (!from.renameTo(to))
@@ -1044,69 +1052,6 @@ public abstract class Storage extends St
+ "-" + Long.toString(storage.getCTime());
}
- String getProperty(Properties props, StorageDirectory sd,
- String name) throws InconsistentFSStateException {
- String property = props.getProperty(name);
- if (property == null) {
- throw new InconsistentFSStateException(sd.root, "file "
- + STORAGE_FILE_VERSION + " has " + name + " missing.");
- }
- return property;
- }
-
- /** Validate and set storage type from {@link Properties}*/
- protected void setStorageType(Properties props, StorageDirectory sd)
- throws InconsistentFSStateException {
- NodeType type = NodeType.valueOf(getProperty(props, sd, "storageType"));
- if (!storageType.equals(type)) {
- throw new InconsistentFSStateException(sd.root,
- "node type is incompatible with others.");
- }
- storageType = type;
- }
-
- /** Validate and set ctime from {@link Properties}*/
- protected void setcTime(Properties props, StorageDirectory sd)
- throws InconsistentFSStateException {
- cTime = Long.parseLong(getProperty(props, sd, "cTime"));
- }
-
- /** Validate and set clusterId from {@link Properties}*/
- protected void setClusterId(Properties props, int layoutVersion,
- StorageDirectory sd) throws InconsistentFSStateException {
- // Set cluster ID in version that supports federation
- if (LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) {
- String cid = getProperty(props, sd, "clusterID");
- if (!(clusterID.equals("") || cid.equals("") || clusterID.equals(cid))) {
- throw new InconsistentFSStateException(sd.getRoot(),
- "cluster Id is incompatible with others.");
- }
- clusterID = cid;
- }
- }
-
- /** Validate and set layout version from {@link Properties}*/
- protected void setLayoutVersion(Properties props, StorageDirectory sd)
- throws IncorrectVersionException, InconsistentFSStateException {
- int lv = Integer.parseInt(getProperty(props, sd, "layoutVersion"));
- if (lv < HdfsConstants.LAYOUT_VERSION) { // future version
- throw new IncorrectVersionException(lv, "storage directory "
- + sd.root.getAbsolutePath());
- }
- layoutVersion = lv;
- }
-
- /** Validate and set namespaceID version from {@link Properties}*/
- protected void setNamespaceID(Properties props, StorageDirectory sd)
- throws InconsistentFSStateException {
- int nsId = Integer.parseInt(getProperty(props, sd, "namespaceID"));
- if (namespaceID != 0 && nsId != 0 && namespaceID != nsId) {
- throw new InconsistentFSStateException(sd.root,
- "namespaceID is incompatible with others.");
- }
- namespaceID = nsId;
- }
-
public static boolean is203LayoutVersion(int layoutVersion) {
for (int lv203 : LAYOUT_VERSIONS_203) {
if (lv203 == layoutVersion) {