You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2017/12/16 02:10:51 UTC
[24/46] hadoop git commit: HDFS-12779. [READ] Allow cluster id to be
specified to the Image generation tool
HDFS-12779. [READ] Allow cluster id to be specified to the Image generation tool
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6cd80b25
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6cd80b25
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6cd80b25
Branch: refs/heads/trunk
Commit: 6cd80b2521e6283036d8c7058d8e452a93ff8e4b
Parents: 90d1b47
Author: Virajith Jalaparti <vi...@apache.org>
Authored: Thu Nov 9 14:09:14 2017 -0800
Committer: Chris Douglas <cd...@apache.org>
Committed: Fri Dec 15 17:51:39 2017 -0800
----------------------------------------------------------------------
.../hdfs/server/protocol/NamespaceInfo.java | 4 ++++
.../hdfs/server/namenode/FileSystemImage.java | 4 ++++
.../hdfs/server/namenode/ImageWriter.java | 11 ++++++++-
.../TestNameNodeProvidedImplementation.java | 24 +++++++++++++++++++-
4 files changed, 41 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cd80b25/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
index 66ce9ee..433d9b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
@@ -160,6 +160,10 @@ public class NamespaceInfo extends StorageInfo {
return state;
}
+ public void setClusterID(String clusterID) {
+ this.clusterID = clusterID;
+ }
+
@Override
public String toString(){
return super.toString() + ";bpid=" + blockPoolID;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cd80b25/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileSystemImage.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileSystemImage.java b/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileSystemImage.java
index 2e57c9f..b66c830 100644
--- a/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileSystemImage.java
+++ b/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileSystemImage.java
@@ -68,6 +68,7 @@ public class FileSystemImage implements Tool {
options.addOption("b", "blockclass", true, "Block output class");
options.addOption("i", "blockidclass", true, "Block resolver class");
options.addOption("c", "cachedirs", true, "Max active dirents");
+ options.addOption("cid", "clusterID", true, "Cluster ID");
options.addOption("h", "help", false, "Print usage");
return options;
}
@@ -112,6 +113,9 @@ public class FileSystemImage implements Tool {
case "c":
opts.cache(Integer.parseInt(o.getValue()));
break;
+ case "cid":
+ opts.clusterID(o.getValue());
+ break;
default:
throw new UnsupportedOperationException("Internal error");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cd80b25/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageWriter.java b/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageWriter.java
index 390bb39..9bd8852 100644
--- a/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageWriter.java
+++ b/hadoop-tools/hadoop-fs2img/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageWriter.java
@@ -126,13 +126,16 @@ public class ImageWriter implements Closeable {
throw new IllegalStateException("Incompatible layout " +
info.getLayoutVersion() + " (expected " + LAYOUT_VERSION);
}
+ // set the cluster id, if given
+ if (opts.clusterID.length() > 0) {
+ info.setClusterID(opts.clusterID);
+ }
stor.format(info);
blockPoolID = info.getBlockPoolID();
}
outdir = new Path(tmp, "current");
out = outfs.create(new Path(outdir, "fsimage_0000000000000000000"));
} else {
- // XXX necessary? writing a NNStorage now...
outdir = null;
outfs = null;
out = opts.outStream;
@@ -517,6 +520,7 @@ public class ImageWriter implements Closeable {
private UGIResolver ugis;
private Class<? extends UGIResolver> ugisClass;
private BlockAliasMap<FileRegion> blocks;
+ private String clusterID;
@SuppressWarnings("rawtypes")
private Class<? extends BlockAliasMap> aliasMap;
@@ -543,6 +547,7 @@ public class ImageWriter implements Closeable {
NullBlockAliasMap.class, BlockAliasMap.class);
blockIdsClass = conf.getClass(BLOCK_RESOLVER_CLASS,
FixedBlockResolver.class, BlockResolver.class);
+ clusterID = "";
}
@Override
@@ -601,6 +606,10 @@ public class ImageWriter implements Closeable {
return this;
}
+ public Options clusterID(String clusterID) {
+ this.clusterID = clusterID;
+ return this;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cd80b25/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java
index 1f6aebb..22f00aa 100644
--- a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java
+++ b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java
@@ -155,11 +155,18 @@ public class TestNameNodeProvidedImplementation {
void createImage(TreeWalk t, Path out,
Class<? extends BlockResolver> blockIdsClass) throws Exception {
+ createImage(t, out, blockIdsClass, "");
+ }
+
+ void createImage(TreeWalk t, Path out,
+ Class<? extends BlockResolver> blockIdsClass, String clusterID)
+ throws Exception {
ImageWriter.Options opts = ImageWriter.defaults();
opts.setConf(conf);
opts.output(out.toString())
.blocks(TextFileRegionAliasMap.class)
- .blockIds(blockIdsClass);
+ .blockIds(blockIdsClass)
+ .clusterID(clusterID);
try (ImageWriter w = new ImageWriter(opts)) {
for (TreePath e : t) {
w.accept(e);
@@ -562,4 +569,19 @@ public class TestNameNodeProvidedImplementation {
dnInfos[0].getDatanodeUuid());
}
}
+
+ @Test
+ public void testSetClusterID() throws Exception {
+ String clusterID = "PROVIDED-CLUSTER";
+ createImage(new FSTreeWalk(NAMEPATH, conf), NNDIRPATH,
+ FixedBlockResolver.class, clusterID);
+ // 2 Datanodes, 1 PROVIDED and other DISK
+ startCluster(NNDIRPATH, 2, null,
+ new StorageType[][] {
+ {StorageType.PROVIDED},
+ {StorageType.DISK}},
+ false);
+ NameNode nn = cluster.getNameNode();
+ assertEquals(clusterID, nn.getNamesystem().getClusterId());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org