You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ar...@apache.org on 2013/11/13 18:15:37 UTC
svn commit: r1541620 - in
/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs: ./
CHANGES.txt src/main/java/
src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
src/test/resources/testCacheAdminConf.xml
Author: arp
Date: Wed Nov 13 17:15:36 2013
New Revision: 1541620
URL: http://svn.apache.org/r1541620
Log:
Merging r1541342 through r1541617 from trunk to branch HDFS-2832
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml
Propchange: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1541342-1541617
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1541620&r1=1541619&r2=1541620&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Nov 13 17:15:36 2013
@@ -192,6 +192,8 @@ Trunk (Unreleased)
HDFS-5450. Better API for getting the cached blocks locations. (wang)
+ HDFS-5485. Add command-line support for modifyDirective. (cmccabe)
+
OPTIMIZATIONS
HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only. (cmccabe)
Propchange: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1541342-1541617
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java?rev=1541620&r1=1541619&r2=1541620&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java Wed Nov 13 17:15:36 2013
@@ -248,6 +248,84 @@ public class CacheAdmin extends Configur
}
}
+ private static class ModifyPathBasedCacheDirectiveCommand implements Command {
+ @Override
+ public String getName() {
+ return "-modifyDirective";
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[" + getName() +
+ " -id <id> [-path <path>] [-replication <replication>] " +
+ "[-pool <pool-name>] ]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ TableListing listing = getOptionDescriptionListing();
+ listing.addRow("<id>", "The ID of the directive to modify (required)");
+ listing.addRow("<path>", "A path to cache. The path can be " +
+ "a directory or a file. (optional)");
+ listing.addRow("<replication>", "The cache replication factor to use. " +
+ "(optional)");
+ listing.addRow("<pool-name>", "The pool to which the directive will be " +
+ "added. You must have write permission on the cache pool "
+ + "in order to move a directive into it. (optional)");
+ return getShortUsage() + "\n" +
+ "Modify a PathBasedCache directive.\n\n" +
+ listing.toString();
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ PathBasedCacheDirective.Builder builder =
+ new PathBasedCacheDirective.Builder();
+ boolean modified = false;
+ String idString = StringUtils.popOptionWithArgument("-id", args);
+ if (idString == null) {
+ System.err.println("You must specify a directive ID with -id.");
+ return 1;
+ }
+ builder.setId(Long.parseLong(idString));
+ String path = StringUtils.popOptionWithArgument("-path", args);
+ if (path != null) {
+ builder.setPath(new Path(path));
+ modified = true;
+ }
+ String replicationString =
+ StringUtils.popOptionWithArgument("-replication", args);
+ if (replicationString != null) {
+ builder.setReplication(Short.parseShort(replicationString));
+ modified = true;
+ }
+ String poolName =
+ StringUtils.popOptionWithArgument("-pool", args);
+ if (poolName != null) {
+ builder.setPool(poolName);
+ modified = true;
+ }
+ if (!args.isEmpty()) {
+ System.err.println("Can't understand argument: " + args.get(0));
+ System.err.println("Usage is " + getShortUsage());
+ return 1;
+ }
+ if (!modified) {
+ System.err.println("No modifications were specified.");
+ return 1;
+ }
+ DistributedFileSystem dfs = getDFS(conf);
+ try {
+ dfs.modifyPathBasedCacheDirective(builder.build());
+ System.out.println("Modified PathBasedCache entry " + idString);
+ } catch (IOException e) {
+ System.err.println(prettifyException(e));
+ return 2;
+ }
+ return 0;
+ }
+ }
+
private static class RemovePathBasedCacheDirectivesCommand implements Command {
@Override
public String getName() {
@@ -352,6 +430,7 @@ public class CacheAdmin extends Configur
TableListing tableListing = new TableListing.Builder().
addField("ID", Justification.LEFT).
addField("POOL", Justification.LEFT).
+ addField("REPLICATION", Justification.LEFT).
addField("PATH", Justification.LEFT).
build();
DistributedFileSystem dfs = getDFS(conf);
@@ -362,6 +441,7 @@ public class CacheAdmin extends Configur
PathBasedCacheDirective directive = iter.next();
String row[] = new String[] {
"" + directive.getId(), directive.getPool(),
+ "" + directive.getReplication(),
directive.getPath().toUri().getPath(),
};
tableListing.addRow(row);
@@ -744,9 +824,10 @@ public class CacheAdmin extends Configur
private static Command[] COMMANDS = {
new AddPathBasedCacheDirectiveCommand(),
+ new ModifyPathBasedCacheDirectiveCommand(),
+ new ListPathBasedCacheDirectiveCommand(),
new RemovePathBasedCacheDirectiveCommand(),
new RemovePathBasedCacheDirectivesCommand(),
- new ListPathBasedCacheDirectiveCommand(),
new AddCachePoolCommand(),
new ModifyCachePoolCommand(),
new RemoveCachePoolCommand(),
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml?rev=1541620&r1=1541619&r2=1541620&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml Wed Nov 13 17:15:36 2013
@@ -180,15 +180,15 @@
</comparator>
<comparator>
<type>SubstringComparator</type>
- <expected-output>1 pool1 /foo</expected-output>
+ <expected-output>1 pool1 1 /foo</expected-output>
</comparator>
<comparator>
<type>SubstringComparator</type>
- <expected-output>2 pool1 /bar</expected-output>
+ <expected-output>2 pool1 1 /bar</expected-output>
</comparator>
<comparator>
<type>SubstringComparator</type>
- <expected-output>3 pool1 /baz</expected-output>
+ <expected-output>3 pool1 2 /baz</expected-output>
</comparator>
</comparators>
</test>
@@ -234,11 +234,11 @@
</comparator>
<comparator>
<type>SubstringComparator</type>
- <expected-output>8 pool2 /baz</expected-output>
+ <expected-output>8 pool2 1 /baz</expected-output>
</comparator>
<comparator>
<type>SubstringComparator</type>
- <expected-output>9 pool2 /buz</expected-output>
+ <expected-output>9 pool2 1 /buz</expected-output>
</comparator>
</comparators>
</test>
@@ -265,11 +265,11 @@
</comparator>
<comparator>
<type>SubstringComparator</type>
- <expected-output>10 pool1 /foo</expected-output>
+ <expected-output>10 pool1 1 /foo</expected-output>
</comparator>
<comparator>
<type>SubstringComparator</type>
- <expected-output>12 pool2 /foo</expected-output>
+ <expected-output>12 pool2 1 /foo</expected-output>
</comparator>
</comparators>
</test>
@@ -296,7 +296,7 @@
</comparator>
<comparator>
<type>SubstringComparator</type>
- <expected-output>16 pool2 /foo</expected-output>
+ <expected-output>16 pool2 1 /foo</expected-output>
</comparator>
</comparators>
</test>
@@ -320,7 +320,7 @@
</comparator>
<comparator>
<type>SubstringComparator</type>
- <expected-output>19 pool1 /bar</expected-output>
+ <expected-output>19 pool1 1 /bar</expected-output>
</comparator>
</comparators>
</test>
@@ -349,11 +349,37 @@
</comparator>
<comparator>
<type>SubstringComparator</type>
- <expected-output>22 pool1 /bar</expected-output>
+ <expected-output>22 pool1 1 /bar</expected-output>
</comparator>
<comparator>
<type>SubstringComparator</type>
- <expected-output>24 pool2 /bar</expected-output>
+ <expected-output>24 pool2 1 /bar</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test> <!--Tested -->
+ <description>Testing modifying directives</description>
+ <test-commands>
+ <cache-admin-command>-addPool pool1</cache-admin-command>
+ <cache-admin-command>-addPool pool2</cache-admin-command>
+ <cache-admin-command>-addDirective -path /foo -pool pool2</cache-admin-command>
+ <cache-admin-command>-modifyDirective -id 25 -path /bar2</cache-admin-command>
+ <cache-admin-command>-modifyDirective -id 25 -pool pool1 -path /bar3</cache-admin-command>
+ <cache-admin-command>-listDirectives -path /bar3</cache-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <cache-admin-command>-removePool pool1</cache-admin-command>
+ <cache-admin-command>-removePool pool2</cache-admin-command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Found 1 entry</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>25 pool1 1 /bar3</expected-output>
</comparator>
</comparators>
</test>