You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ji...@apache.org on 2014/08/09 00:21:39 UTC
svn commit: r1616897 [4/4] - in
/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project:
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/
hadoop-hdfs-httpfs/src/main/jav...
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_write.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_write.c?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_write.c (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_write.c Fri Aug 8 22:21:35 2014
@@ -21,23 +21,31 @@
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
+#include <sys/types.h>
int main(int argc, char **argv) {
+ hdfsFS fs;
+ const char *writeFileName = argv[1];
+ off_t fileTotalSize = strtoul(argv[2], NULL, 10);
+ long long tmpBufferSize = strtoul(argv[3], NULL, 10);
+ tSize bufferSize;
+ hdfsFile writeFile;
+ char* buffer;
+ int i;
+ off_t nrRemaining;
+ tSize curSize;
+ tSize written;
if (argc != 4) {
fprintf(stderr, "Usage: hdfs_write <filename> <filesize> <buffersize>\n");
exit(-1);
}
- hdfsFS fs = hdfsConnect("default", 0);
+ fs = hdfsConnect("default", 0);
if (!fs) {
fprintf(stderr, "Oops! Failed to connect to hdfs!\n");
exit(-1);
}
-
- const char* writeFileName = argv[1];
- off_t fileTotalSize = strtoul(argv[2], NULL, 10);
- long long tmpBufferSize = strtoul(argv[3], NULL, 10);
// sanity check
if(fileTotalSize == ULONG_MAX && errno == ERANGE) {
@@ -51,30 +59,27 @@ int main(int argc, char **argv) {
exit(-3);
}
- tSize bufferSize = tmpBufferSize;
+ bufferSize = (tSize)tmpBufferSize;
- hdfsFile writeFile = hdfsOpenFile(fs, writeFileName, O_WRONLY, bufferSize, 0, 0);
+ writeFile = hdfsOpenFile(fs, writeFileName, O_WRONLY, bufferSize, 0, 0);
if (!writeFile) {
fprintf(stderr, "Failed to open %s for writing!\n", writeFileName);
exit(-2);
}
// data to be written to the file
- char* buffer = malloc(sizeof(char) * bufferSize);
+ buffer = malloc(sizeof(char) * bufferSize);
if(buffer == NULL) {
fprintf(stderr, "Could not allocate buffer of size %d\n", bufferSize);
return -2;
}
- int i = 0;
for (i=0; i < bufferSize; ++i) {
buffer[i] = 'a' + (i%26);
}
// write to the file
- off_t nrRemaining;
for (nrRemaining = fileTotalSize; nrRemaining > 0; nrRemaining -= bufferSize ) {
- tSize curSize = ( bufferSize < nrRemaining ) ? bufferSize : (tSize)nrRemaining;
- tSize written;
+ curSize = ( bufferSize < nrRemaining ) ? bufferSize : (tSize)nrRemaining;
if ((written = hdfsWrite(fs, writeFile, (void*)buffer, curSize)) != curSize) {
fprintf(stderr, "ERROR: hdfsWrite returned an error on write: %d\n", written);
exit(-3);
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c Fri Aug 8 22:21:35 2014
@@ -19,12 +19,12 @@
#include "expect.h"
#include "hdfs.h"
#include "native_mini_dfs.h"
+#include "platform.h"
#include <errno.h>
#include <inttypes.h>
-#include <semaphore.h>
-#include <pthread.h>
#include <unistd.h>
+#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -53,7 +53,7 @@ static uint8_t *getZeroCopyBlockData(int
exit(1);
}
for (i = 0; i < TEST_ZEROCOPY_FULL_BLOCK_SIZE; i++) {
- buf[i] = blockIdx + (i % 17);
+ buf[i] = (uint8_t)(blockIdx + (i % 17));
}
return buf;
}
@@ -69,18 +69,6 @@ static int getZeroCopyBlockLen(int block
}
}
-static void printBuf(const uint8_t *buf, size_t len) __attribute__((unused));
-
-static void printBuf(const uint8_t *buf, size_t len)
-{
- size_t i;
-
- for (i = 0; i < len; i++) {
- fprintf(stderr, "%02x", buf[i]);
- }
- fprintf(stderr, "\n");
-}
-
static int doTestZeroCopyReads(hdfsFS fs, const char *fileName)
{
hdfsFile file = NULL;
@@ -127,8 +115,9 @@ static int doTestZeroCopyReads(hdfsFS fs
EXPECT_NONNULL(block);
EXPECT_ZERO(memcmp(block, hadoopRzBufferGet(buffer), SMALL_READ_LEN));
hadoopRzBufferFree(file, buffer);
- EXPECT_INT_EQ(TEST_ZEROCOPY_FULL_BLOCK_SIZE + SMALL_READ_LEN,
- hdfsTell(fs, file));
+ EXPECT_INT64_EQ(
+ (int64_t)TEST_ZEROCOPY_FULL_BLOCK_SIZE + (int64_t)SMALL_READ_LEN,
+ hdfsTell(fs, file));
EXPECT_ZERO(expectFileStats(file,
TEST_ZEROCOPY_FULL_BLOCK_SIZE + SMALL_READ_LEN,
TEST_ZEROCOPY_FULL_BLOCK_SIZE + SMALL_READ_LEN,
@@ -165,7 +154,7 @@ static int doTestZeroCopyReads(hdfsFS fs
free(block);
block = getZeroCopyBlockData(2);
EXPECT_NONNULL(block);
- EXPECT_ZERO(memcmp(block, hadoopRzBufferGet(buffer) +
+ EXPECT_ZERO(memcmp(block, (uint8_t*)hadoopRzBufferGet(buffer) +
(TEST_ZEROCOPY_FULL_BLOCK_SIZE - SMALL_READ_LEN), SMALL_READ_LEN));
hadoopRzBufferFree(file, buffer);
@@ -219,8 +208,10 @@ int main(void)
{
int port;
struct NativeMiniDfsConf conf = {
- .doFormat = 1,
- .configureShortCircuit = 1,
+ 1, /* doFormat */
+ 0, /* webhdfsEnabled */
+ 0, /* namenodeHttpPort */
+ 1, /* configureShortCircuit */
};
char testFileName[TEST_FILE_NAME_LENGTH];
hdfsFS fs;
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c Fri Aug 8 22:21:35 2014
@@ -19,11 +19,11 @@
#include "expect.h"
#include "hdfs.h"
#include "native_mini_dfs.h"
+#include "os/thread.h"
#include <errno.h>
#include <inttypes.h>
-#include <semaphore.h>
-#include <pthread.h>
+#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -35,8 +35,6 @@
#define TLH_DEFAULT_BLOCK_SIZE 134217728
-static sem_t tlhSem;
-
static struct NativeMiniDfsCluster* tlhCluster;
struct tlhThreadInfo {
@@ -44,18 +42,19 @@ struct tlhThreadInfo {
int threadIdx;
/** 0 = thread was successful; error code otherwise */
int success;
- /** pthread identifier */
- pthread_t thread;
+ /** thread identifier */
+ thread theThread;
};
static int hdfsSingleNameNodeConnect(struct NativeMiniDfsCluster *cl, hdfsFS *fs,
const char *username)
{
- int ret, port;
+ int ret;
+ tPort port;
hdfsFS hdfs;
struct hdfsBuilder *bld;
- port = nmdGetNameNodePort(cl);
+ port = (tPort)nmdGetNameNodePort(cl);
if (port < 0) {
fprintf(stderr, "hdfsSingleNameNodeConnect: nmdGetNameNodePort "
"returned error %d\n", port);
@@ -164,7 +163,7 @@ static int doTestHdfsOperations(struct t
EXPECT_NONNULL(file);
/* TODO: implement writeFully and use it here */
- expected = strlen(paths->prefix);
+ expected = (int)strlen(paths->prefix);
ret = hdfsWrite(fs, file, paths->prefix, expected);
if (ret < 0) {
ret = errno;
@@ -186,9 +185,9 @@ static int doTestHdfsOperations(struct t
EXPECT_ZERO(hdfsFileGetReadStatistics(file, &readStats));
errno = 0;
- EXPECT_ZERO(readStats->totalBytesRead);
- EXPECT_ZERO(readStats->totalLocalBytesRead);
- EXPECT_ZERO(readStats->totalShortCircuitBytesRead);
+ EXPECT_UINT64_EQ(UINT64_C(0), readStats->totalBytesRead);
+ EXPECT_UINT64_EQ(UINT64_C(0), readStats->totalLocalBytesRead);
+ EXPECT_UINT64_EQ(UINT64_C(0), readStats->totalShortCircuitBytesRead);
hdfsFileFreeReadStatistics(readStats);
/* TODO: implement readFully and use it here */
ret = hdfsRead(fs, file, tmp, sizeof(tmp));
@@ -204,7 +203,7 @@ static int doTestHdfsOperations(struct t
}
EXPECT_ZERO(hdfsFileGetReadStatistics(file, &readStats));
errno = 0;
- EXPECT_INT_EQ(expected, readStats->totalBytesRead);
+ EXPECT_UINT64_EQ((uint64_t)expected, readStats->totalBytesRead);
hdfsFileFreeReadStatistics(readStats);
EXPECT_ZERO(memcmp(paths->prefix, tmp, expected));
EXPECT_ZERO(hdfsCloseFile(fs, file));
@@ -262,12 +261,11 @@ static int testHdfsOperationsImpl(struct
return 0;
}
-static void *testHdfsOperations(void *v)
+static void testHdfsOperations(void *v)
{
struct tlhThreadInfo *ti = (struct tlhThreadInfo*)v;
int ret = testHdfsOperationsImpl(ti);
ti->success = ret;
- return NULL;
}
static int checkFailures(struct tlhThreadInfo *ti, int tlhNumThreads)
@@ -304,7 +302,7 @@ int main(void)
const char *tlhNumThreadsStr;
struct tlhThreadInfo ti[TLH_MAX_THREADS];
struct NativeMiniDfsConf conf = {
- .doFormat = 1,
+ 1, /* doFormat */
};
tlhNumThreadsStr = getenv("TLH_NUM_THREADS");
@@ -323,21 +321,20 @@ int main(void)
ti[i].threadIdx = i;
}
- EXPECT_ZERO(sem_init(&tlhSem, 0, tlhNumThreads));
tlhCluster = nmdCreate(&conf);
EXPECT_NONNULL(tlhCluster);
EXPECT_ZERO(nmdWaitClusterUp(tlhCluster));
for (i = 0; i < tlhNumThreads; i++) {
- EXPECT_ZERO(pthread_create(&ti[i].thread, NULL,
- testHdfsOperations, &ti[i]));
+ ti[i].theThread.start = testHdfsOperations;
+ ti[i].theThread.arg = &ti[i];
+ EXPECT_ZERO(threadCreate(&ti[i].theThread));
}
for (i = 0; i < tlhNumThreads; i++) {
- EXPECT_ZERO(pthread_join(ti[i].thread, NULL));
+ EXPECT_ZERO(threadJoin(&ti[i].theThread));
}
EXPECT_ZERO(nmdShutdown(tlhCluster));
nmdFree(tlhCluster);
- EXPECT_ZERO(sem_destroy(&tlhSem));
return checkFailures(ti, tlhNumThreads);
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_native_mini_dfs.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_native_mini_dfs.c?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_native_mini_dfs.c (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_native_mini_dfs.c Fri Aug 8 22:21:35 2014
@@ -22,7 +22,7 @@
#include <errno.h>
static struct NativeMiniDfsConf conf = {
- .doFormat = 1,
+ 1, /* doFormat */
};
/**
Propchange: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1614232-1616893
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html Fri Aug 8 22:21:35 2014
@@ -281,7 +281,7 @@
{#DeadNodes}
<tr class="danger">
<td>{name} ({xferaddr})</td>
- <td>{lastContact}</td>
+ <td>{#helper_lastcontact_tostring value="{lastContact}"/}</td>
<td>Dead{?decommissioned}, Decommissioned{/decommissioned}</td>
<td>-</td>
<td>-</td>
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js Fri Aug 8 22:21:35 2014
@@ -139,6 +139,14 @@
}
function load_datanode_info() {
+
+ var HELPERS = {
+ 'helper_lastcontact_tostring' : function (chunk, ctx, bodies, params) {
+ var value = dust.helpers.tap(params.value, chunk, ctx);
+ return chunk.write('' + new Date(Date.now()-1000*Number(value)));
+ }
+ };
+
function workaround(r) {
function node_map_to_array(nodes) {
var res = [];
@@ -160,7 +168,8 @@
'/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo',
guard_with_startup_progress(function (resp) {
var data = workaround(resp.beans[0]);
- dust.render('datanode-info', data, function(err, out) {
+ var base = dust.makeBase(HELPERS);
+ dust.render('datanode-info', base.push(data), function(err, out) {
$('#tab-datanode').html(out);
$('#ui-tabs a[href="#tab-datanode"]').tab('show');
});
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm Fri Aug 8 22:21:35 2014
@@ -143,8 +143,8 @@ HDFS Users Guide
** DFSAdmin Command
- The <<<bin/hadoop dfsadmin>>> command supports a few HDFS administration
- related operations. The <<<bin/hadoop dfsadmin -help>>> command lists all the
+ The <<<bin/hdfs dfsadmin>>> command supports a few HDFS administration
+ related operations. The <<<bin/hdfs dfsadmin -help>>> command lists all the
commands currently supported. For e.g.:
* <<<-report>>>: reports basic statistics of HDFS. Some of this
@@ -172,7 +172,7 @@ HDFS Users Guide
of racks and datanodes attached to the tracks as viewed by the
NameNode.
- For command usage, see {{{../hadoop-common/CommandsManual.html#dfsadmin}dfsadmin}}.
+ For command usage, see {{{./HDFSCommands.html#dfsadmin}dfsadmin}}.
* Secondary NameNode
@@ -207,7 +207,7 @@ HDFS Users Guide
primary NameNode if necessary.
For command usage,
- see {{{../hadoop-common/CommandsManual.html#secondarynamenode}secondarynamenode}}.
+ see {{{./HDFSCommands.html#secondarynamenode}secondarynamenode}}.
* Checkpoint Node
@@ -249,7 +249,7 @@ HDFS Users Guide
Multiple checkpoint nodes may be specified in the cluster configuration
file.
- For command usage, see {{{../hadoop-common/CommandsManual.html#namenode}namenode}}.
+ For command usage, see {{{./HDFSCommands.html#namenode}namenode}}.
* Backup Node
@@ -291,7 +291,7 @@ HDFS Users Guide
For a complete discussion of the motivation behind the creation of the
Backup node and Checkpoint node, see {{{https://issues.apache.org/jira/browse/HADOOP-4539}HADOOP-4539}}.
- For command usage, see {{{../hadoop-common/CommandsManual.html#namenode}namenode}}.
+ For command usage, see {{{./HDFSCommands.html#namenode}namenode}}.
* Import Checkpoint
@@ -314,7 +314,7 @@ HDFS Users Guide
verifies that the image in <<<dfs.namenode.checkpoint.dir>>> is consistent,
but does not modify it in any way.
- For command usage, see {{{../hadoop-common/CommandsManual.html#namenode}namenode}}.
+ For command usage, see {{{./HDFSCommands.html#namenode}namenode}}.
* Balancer
@@ -341,7 +341,7 @@ HDFS Users Guide
A brief administrator's guide for balancer is available at
{{{https://issues.apache.org/jira/browse/HADOOP-1652}HADOOP-1652}}.
- For command usage, see {{{../hadoop-common/CommandsManual.html#balancer}balancer}}.
+ For command usage, see {{{./HDFSCommands.html#balancer}balancer}}.
* Rack Awareness
@@ -368,7 +368,7 @@ HDFS Users Guide
allow any modifications to file system or blocks. Normally the NameNode
leaves Safemode automatically after the DataNodes have reported that
most file system blocks are available. If required, HDFS could be
- placed in Safemode explicitly using <<<bin/hadoop dfsadmin -safemode>>>
+ placed in Safemode explicitly using <<<bin/hdfs dfsadmin -safemode>>>
command. NameNode front page shows whether Safemode is on or off. A
more detailed description and configuration is maintained as JavaDoc
for <<<setSafeMode()>>>.
@@ -383,8 +383,8 @@ HDFS Users Guide
most of the recoverable failures. By default fsck ignores open files
but provides an option to select all files during reporting. The HDFS
fsck command is not a Hadoop shell command. It can be run as
- <<<bin/hadoop fsck>>>. For command usage, see
- {{{../hadoop-common/CommandsManual.html#fsck}fsck}}. fsck can be run on
+ <<<bin/hdfs fsck>>>. For command usage, see
+ {{{./HDFSCommands.html#fsck}fsck}}. fsck can be run on
the whole file system or on a subset of files.
* fetchdt
@@ -395,11 +395,11 @@ HDFS Users Guide
Utility uses either RPC or HTTPS (over Kerberos) to get the token, and
thus requires kerberos tickets to be present before the run (run kinit
to get the tickets). The HDFS fetchdt command is not a Hadoop shell
- command. It can be run as <<<bin/hadoop fetchdt DTfile>>>. After you got
+ command. It can be run as <<<bin/hdfs fetchdt DTfile>>>. After you got
the token you can run an HDFS command without having Kerberos tickets,
by pointing <<<HADOOP_TOKEN_FILE_LOCATION>>> environmental variable to the
delegation token file. For command usage, see
- {{{../hadoop-common/CommandsManual.html#fetchdt}fetchdt}} command.
+ {{{./HDFSCommands.html#fetchdt}fetchdt}} command.
* Recovery Mode
@@ -533,5 +533,4 @@ HDFS Users Guide
* Explore {{{./hdfs-default.xml}hdfs-default.xml}}. It includes
brief description of most of the configuration variables available.
- * {{{../hadoop-common/CommandsManual.html}Hadoop Commands Guide}}:
- Hadoop commands usage.
+ * {{{./HDFSCommands.html}HDFS Commands Guide}}: HDFS commands usage.
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java Fri Aug 8 22:21:35 2014
@@ -89,7 +89,7 @@ public class TestBalancer {
private static final Random r = new Random();
static {
- Balancer.setBlockMoveWaitTime(1000L) ;
+ Dispatcher.setBlockMoveWaitTime(1000L) ;
}
static void initConf(Configuration conf) {
@@ -305,12 +305,12 @@ public class TestBalancer {
for (DatanodeInfo datanode : datanodeReport) {
double nodeUtilization = ((double)datanode.getDfsUsed())
/ datanode.getCapacity();
- if (Balancer.Util.shouldBeExcluded(p.nodesToBeExcluded, datanode)) {
+ if (Dispatcher.Util.isExcluded(p.nodesToBeExcluded, datanode)) {
assertTrue(nodeUtilization == 0);
actualExcludedNodeCount++;
continue;
}
- if (!Balancer.Util.shouldBeIncluded(p.nodesToBeIncluded, datanode)) {
+ if (!Dispatcher.Util.isIncluded(p.nodesToBeIncluded, datanode)) {
assertTrue(nodeUtilization == 0);
actualExcludedNodeCount++;
continue;
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java Fri Aug 8 22:21:35 2014
@@ -44,7 +44,7 @@ public class TestBalancerWithHANameNodes
ClientProtocol client;
static {
- Balancer.setBlockMoveWaitTime(1000L);
+ Dispatcher.setBlockMoveWaitTime(1000L);
}
/**
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java Fri Aug 8 22:21:35 2014
@@ -73,7 +73,7 @@ public class TestBalancerWithMultipleNam
private static final Random RANDOM = new Random();
static {
- Balancer.setBlockMoveWaitTime(1000L) ;
+ Dispatcher.setBlockMoveWaitTime(1000L) ;
}
/** Common objects used in various methods. */
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java Fri Aug 8 22:21:35 2014
@@ -75,7 +75,7 @@ public class TestBalancerWithNodeGroup {
static final int DEFAULT_BLOCK_SIZE = 100;
static {
- Balancer.setBlockMoveWaitTime(1000L) ;
+ Dispatcher.setBlockMoveWaitTime(1000L) ;
}
static Configuration createConf() {
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Fri Aug 8 22:21:35 2014
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
@@ -1083,6 +1084,11 @@ public class SimulatedFSDataset implemen
}
@Override
+ public void addVolumes(Collection<StorageLocation> volumes) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public DatanodeStorage getStorage(final String storageUuid) {
return storageUuid.equals(storage.getStorageUuid()) ?
storage.dnStorage :
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java Fri Aug 8 22:21:35 2014
@@ -94,6 +94,8 @@ public class TestFSNamesystemMBean {
"SnapshotStats"));
Long MaxObjects = (Long) (mbs.getAttribute(mxbeanNameFsns,
"MaxObjects"));
+ Integer numStaleStorages = (Integer) (mbs.getAttribute(
+ mxbeanNameFsns, "NumStaleStorages"));
// Metrics that belong to "NameNodeInfo".
// These are metrics that FSNamesystem registers directly with MBeanServer.
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java Fri Aug 8 22:21:35 2014
@@ -30,9 +30,13 @@ import javax.management.MBeanServer;
import javax.management.ObjectName;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIO.POSIX.NoMlockCacheManipulator;
import org.apache.hadoop.util.VersionInfo;
@@ -58,11 +62,14 @@ public class TestNameNodeMXBean {
public void testNameNodeMXBeanInfo() throws Exception {
Configuration conf = new Configuration();
conf.setLong(DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY,
- NativeIO.POSIX.getCacheManipulator().getMemlockLimit());
+ NativeIO.POSIX.getCacheManipulator().getMemlockLimit());
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1);
+
MiniDFSCluster cluster = null;
try {
- cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
cluster.waitActive();
FSNamesystem fsn = cluster.getNameNode().namesystem;
@@ -70,6 +77,29 @@ public class TestNameNodeMXBean {
MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName(
"Hadoop:service=NameNode,name=NameNodeInfo");
+
+ // Define include file to generate deadNodes metrics
+ FileSystem localFileSys = FileSystem.getLocal(conf);
+ Path workingDir = localFileSys.getWorkingDirectory();
+ Path dir = new Path(workingDir,
+ "build/test/data/temp/TestNameNodeMXBean");
+ Path includeFile = new Path(dir, "include");
+ assertTrue(localFileSys.mkdirs(dir));
+ StringBuilder includeHosts = new StringBuilder();
+ for(DataNode dn : cluster.getDataNodes()) {
+ includeHosts.append(dn.getDisplayName()).append("\n");
+ }
+ DFSTestUtil.writeFile(localFileSys, includeFile, includeHosts.toString());
+ conf.set(DFSConfigKeys.DFS_HOSTS, includeFile.toUri().getPath());
+ fsn.getBlockManager().getDatanodeManager().refreshNodes(conf);
+
+ cluster.stopDataNode(0);
+ while (fsn.getNumDatanodesInService() != 2) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {}
+ }
+
// get attribute "ClusterId"
String clusterId = (String) mbs.getAttribute(mxbeanName, "ClusterId");
assertEquals(fsn.getClusterId(), clusterId);
@@ -121,6 +151,15 @@ public class TestNameNodeMXBean {
String deadnodeinfo = (String) (mbs.getAttribute(mxbeanName,
"DeadNodes"));
assertEquals(fsn.getDeadNodes(), deadnodeinfo);
+ Map<String, Map<String, Object>> deadNodes =
+ (Map<String, Map<String, Object>>) JSON.parse(deadnodeinfo);
+ assertTrue(deadNodes.size() > 0);
+ for (Map<String, Object> deadNode : deadNodes.values()) {
+ assertTrue(deadNode.containsKey("lastContact"));
+ assertTrue(deadNode.containsKey("decommissioned"));
+ assertTrue(deadNode.containsKey("xferaddr"));
+ }
+
// get attribute NodeUsage
String nodeUsage = (String) (mbs.getAttribute(mxbeanName,
"NodeUsage"));
@@ -181,7 +220,7 @@ public class TestNameNodeMXBean {
assertEquals(1, statusMap.get("active").size());
assertEquals(1, statusMap.get("failed").size());
assertEquals(0L, mbs.getAttribute(mxbeanName, "CacheUsed"));
- assertEquals(NativeIO.POSIX.getCacheManipulator().getMemlockLimit() *
+ assertEquals(NativeIO.POSIX.getCacheManipulator().getMemlockLimit() *
cluster.getDataNodes().size(),
mbs.getAttribute(mxbeanName, "CacheCapacity"));
} finally {
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java Fri Aug 8 22:21:35 2014
@@ -26,6 +26,7 @@ import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
+import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.net.URI;
import java.util.ArrayList;
@@ -49,6 +50,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
@@ -64,6 +66,9 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
/**
* Startup and checkpoint tests
*
@@ -684,4 +689,40 @@ public class TestStartup {
}
}
}
+
+
+ /**
+ * Verify the following scenario.
+ * 1. NN restarts.
+ * 2. Heartbeat RPC will retry and succeed. NN asks DN to reregister.
+ * 3. After reregistration completes, DN will send Heartbeat, followed by
+ * Blockreport.
+ * 4. NN will mark DatanodeStorageInfo#blockContentsStale to false.
+ * @throws Exception
+ */
+ @Test(timeout = 60000)
+ public void testStorageBlockContentsStaleAfterNNRestart() throws Exception {
+ MiniDFSCluster dfsCluster = null;
+ try {
+ Configuration config = new Configuration();
+ dfsCluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build();
+ dfsCluster.waitActive();
+ dfsCluster.restartNameNode(true);
+ BlockManagerTestUtil.checkHeartbeat(
+ dfsCluster.getNamesystem().getBlockManager());
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ ObjectName mxbeanNameFsns = new ObjectName(
+ "Hadoop:service=NameNode,name=FSNamesystemState");
+ Integer numStaleStorages = (Integer) (mbs.getAttribute(
+ mxbeanNameFsns, "NumStaleStorages"));
+ assertEquals(0, numStaleStorages.intValue());
+ } finally {
+ if (dfsCluster != null) {
+ dfsCluster.shutdown();
+ }
+ }
+
+ return;
+ }
+
}