You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2012/09/10 20:45:53 UTC
svn commit: r1383030 [1/2] - in
/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project: hadoop-hdfs-httpfs/
hadoop-hdfs/ hadoop-hdfs/src/ hadoop-hdfs/src/contrib/bkjournal/
hadoop-hdfs/src/contrib/bkjournal/dev-support/
hadoop-hdfs/src/contrib/bkjournal...
Author: todd
Date: Mon Sep 10 18:45:45 2012
New Revision: 1383030
URL: http://svn.apache.org/viewvc?rev=1383030&view=rev
Log:
Merge trunk into QJM branch
Added:
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/dev-support/
- copied from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/dev-support/
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/dev-support/findbugsExcludeFile.xml
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/dev-support/findbugsExcludeFile.xml
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/proto/
- copied from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/proto/
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/proto/bkjournal.proto
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/proto/bkjournal.proto
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/
- copied from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/
- copied from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/
- copied from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/exception.c
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/exception.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/exception.h
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/exception.h
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/expect.h
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/expect.h
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.c
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.h
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.h
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.c
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.h
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.h
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_jni.c
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_jni.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.h
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.h
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_web.c
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_web.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/jni_helper.c
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/jni_helper.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/jni_helper.h
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/jni_helper.h
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/test_libwebhdfs_multi_write.c
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/test_libwebhdfs_multi_write.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/test_libwebhdfs_ops.c
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/test_libwebhdfs_ops.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/test_libwebhdfs_read.c
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/test_libwebhdfs_read.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/test_libwebhdfs_threaded.c
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/test_libwebhdfs_threaded.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/test_libwebhdfs_write.c
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/test_libwebhdfs_write.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/test_read_bm.c
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/test_read_bm.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/webhdfs.h
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/webhdfs.h
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSetUMask.java
- copied unchanged from r1383029, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSetUMask.java
Modified:
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/pom.xml
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSTrash.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestMultipleNNDataBlockScanner.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTransferFsImage.java
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1380987-1383029
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml Mon Sep 10 18:45:45 2012
@@ -104,10 +104,6 @@
<artifactId>stax-api</artifactId>
</exclusion>
<exclusion>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </exclusion>
- <exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Sep 10 18:45:45 2012
@@ -1,6 +1,6 @@
Hadoop HDFS Change Log
-Trunk (unreleased changes)
+Trunk (Unreleased)
INCOMPATIBLE CHANGES
@@ -215,7 +215,22 @@ Trunk (unreleased changes)
HDFS-3678. Edit log files are never being purged from 2NN. (atm)
-Branch-2 ( Unreleased changes )
+Release 2.0.3-alpha - Unreleased
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ HDFS-2656. Add libwebhdfs, a pure C client based on WebHDFS.
+ (Jaimin D Jetly and Jing Zhao via szetszwo)
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@@ -723,6 +738,18 @@ Branch-2 ( Unreleased changes )
HDFS-3469. start-dfs.sh will start zkfc, but stop-dfs.sh will not stop zkfc similarly.
(Vinay via umamahesh)
+
+ HDFS-1490. TransferFSImage should timeout (Dmytro Molkov and Vinay via todd)
+
+ HDFS-3828. Block Scanner rescans blocks too frequently.
+ (Andy Isaacson via eli)
+
+ HDFS-3809. Make BKJM use protobufs for all serialization with ZK.(Ivan Kelly via umamahesh)
+
+ HDFS-3895. hadoop-client must include commons-cli (tucu)
+
+ HDFS-2757. Cannot read a local block that's being written to when
+ using the local read short circuit. (Jean-Daniel Cryans via eli)
BREAKDOWN OF HDFS-3042 SUBTASKS
@@ -1623,6 +1650,9 @@ Release 0.23.3 - UNRELEASED
HDFS-3852. TestHftpDelegationToken is broken after HADOOP-8225 (daryn)
+ HDFS-3890. filecontext mkdirs doesn't apply umask as expected
+ (Tom Graves via daryn)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/pom.xml?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/pom.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/pom.xml Mon Sep 10 18:45:45 2012
@@ -35,6 +35,7 @@ http://maven.apache.org/xsd/maven-4.0.0.
<kdc.resource.dir>../../hadoop-common-project/hadoop-common/src/test/resources/kdc</kdc.resource.dir>
<is.hadoop.component>true</is.hadoop.component>
<require.fuse>false</require.fuse>
+ <require.libwebhdfs>false</require.libwebhdfs>
</properties>
<dependencies>
@@ -495,7 +496,7 @@ http://maven.apache.org/xsd/maven-4.0.0.
<mkdir dir="${project.build.directory}/native"/>
<exec executable="cmake" dir="${project.build.directory}/native"
failonerror="true">
- <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_FUSE=${require.fuse}"/>
+ <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse}"/>
</exec>
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
<arg line="VERBOSE=1"/>
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt Mon Sep 10 18:45:45 2012
@@ -147,4 +147,7 @@ target_link_libraries(test_libhdfs_threa
pthread
)
+IF(REQUIRE_LIBWEBHDFS)
+ add_subdirectory(contrib/libwebhdfs)
+ENDIF(REQUIRE_LIBWEBHDFS)
add_subdirectory(main/native/fuse-dfs)
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml Mon Sep 10 18:45:45 2012
@@ -89,6 +89,90 @@ http://maven.apache.org/xsd/maven-4.0.0.
<scope>test</scope>
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>${project.build.directory}/generated-sources/java</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <configuration>
+ <skipTests>false</skipTests>
+ </configuration>
+ <executions>
+ <execution>
+ <id>compile-proto</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <echo file="target/compile-proto.sh">
+ PROTO_DIR=src/main/proto
+ INCLUDE_DIR=../../main/proto
+ JAVA_DIR=target/generated-sources/java
+ which cygpath 2> /dev/null
+ if [ $? = 1 ]; then
+ IS_WIN=false
+ else
+ IS_WIN=true
+ WIN_PROTO_DIR=`cygpath --windows $PROTO_DIR`
+ WIN_JAVA_DIR=`cygpath --windows $JAVA_DIR`
+ WIN_INCLUDE_DIR=`cygpath --windows $INCLUDE_DIR`
+ fi
+ mkdir -p $JAVA_DIR 2> /dev/null
+ for PROTO_FILE in `ls $PROTO_DIR/*.proto 2> /dev/null`
+ do
+ if [ "$IS_WIN" = "true" ]; then
+ protoc -I$WIN_PROTO_DIR -I$WIN_INCLUDE_DIR --java_out=$WIN_JAVA_DIR $PROTO_FILE
+ else
+ protoc -I$PROTO_DIR -I$INCLUDE_DIR --java_out=$JAVA_DIR $PROTO_FILE
+ fi
+ done
+ </echo>
+ <exec executable="sh" dir="${basedir}" failonerror="true">
+ <arg line="target/compile-proto.sh"/>
+ </exec>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>dev-support/findbugsExcludeFile.xml</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
<profiles>
<profile>
<id>dist</id>
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java Mon Sep 10 18:45:45 2012
@@ -70,7 +70,7 @@ class BookKeeperEditLogInputStream exten
this.lh = lh;
this.firstTxId = metadata.getFirstTxId();
this.lastTxId = metadata.getLastTxId();
- this.logVersion = metadata.getVersion();
+ this.logVersion = metadata.getDataLayoutVersion();
this.inProgress = metadata.isInProgress();
if (firstBookKeeperEntry < 0
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java Mon Sep 10 18:45:45 2012
@@ -50,6 +50,11 @@ import java.io.IOException;
import java.net.URI;
+import org.apache.hadoop.hdfs.protocolPB.PBHelper;
+import org.apache.hadoop.contrib.bkjournal.BKJournalProtos.VersionProto;
+import com.google.protobuf.TextFormat;
+import static com.google.common.base.Charsets.UTF_8;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.google.common.annotations.VisibleForTesting;
@@ -143,36 +148,17 @@ public class BookKeeperJournalManager im
private final int quorumSize;
private final String digestpw;
private final CountDownLatch zkConnectLatch;
-
+ private final NamespaceInfo nsInfo;
private LedgerHandle currentLedger = null;
- private int bytesToInt(byte[] b) {
- assert b.length >= 4;
- return b[0] << 24 | b[1] << 16 | b[2] << 8 | b[3];
- }
-
- private byte[] intToBytes(int i) {
- return new byte[] {
- (byte)(i >> 24),
- (byte)(i >> 16),
- (byte)(i >> 8),
- (byte)(i) };
- }
-
- BookKeeperJournalManager(Configuration conf, URI uri) throws IOException {
- this(conf, uri, null);
- // TODO(ivank): update BookKeeperJournalManager to do something
- // with the NamespaceInfo. This constructor has been added
- // for compatibility with the old tests, and may be removed
- // when the tests are updated.
- }
-
/**
* Construct a Bookkeeper journal manager.
*/
public BookKeeperJournalManager(Configuration conf, URI uri,
NamespaceInfo nsInfo) throws IOException {
this.conf = conf;
+ this.nsInfo = nsInfo;
+
String zkConnect = uri.getAuthority().replace(";", ",");
String zkPath = uri.getPath();
ensembleSize = conf.getInt(BKJM_BOOKKEEPER_ENSEMBLE_SIZE,
@@ -202,10 +188,32 @@ public class BookKeeperJournalManager im
Stat versionStat = zkc.exists(versionPath, false);
if (versionStat != null) {
byte[] d = zkc.getData(versionPath, false, versionStat);
+ VersionProto.Builder builder = VersionProto.newBuilder();
+ TextFormat.merge(new String(d, UTF_8), builder);
+ if (!builder.isInitialized()) {
+ throw new IOException("Invalid/Incomplete data in znode");
+ }
+ VersionProto vp = builder.build();
+
// There's only one version at the moment
- assert bytesToInt(d) == BKJM_LAYOUT_VERSION;
- } else {
- zkc.create(versionPath, intToBytes(BKJM_LAYOUT_VERSION),
+ assert vp.getLayoutVersion() == BKJM_LAYOUT_VERSION;
+
+ NamespaceInfo readns = PBHelper.convert(vp.getNamespaceInfo());
+
+ if (nsInfo.getNamespaceID() != readns.getNamespaceID() ||
+ !nsInfo.clusterID.equals(readns.getClusterID()) ||
+ !nsInfo.getBlockPoolID().equals(readns.getBlockPoolID())) {
+ String err = String.format("Environment mismatch. Running process %s"
+ +", stored in ZK %s", nsInfo, readns);
+ LOG.error(err);
+ throw new IOException(err);
+ }
+ } else if (nsInfo.getNamespaceID() > 0) {
+ VersionProto.Builder builder = VersionProto.newBuilder();
+ builder.setNamespaceInfo(PBHelper.convert(nsInfo))
+ .setLayoutVersion(BKJM_LAYOUT_VERSION);
+ byte[] data = TextFormat.printToString(builder.build()).getBytes(UTF_8);
+ zkc.create(versionPath, data,
Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
}
@@ -214,11 +222,11 @@ public class BookKeeperJournalManager im
Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
}
prepareBookKeeperEnv();
- bkc = new BookKeeper(new ClientConfiguration(),
- zkc);
+ bkc = new BookKeeper(new ClientConfiguration(), zkc);
} catch (KeeperException e) {
throw new IOException("Error initializing zk", e);
} catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
throw new IOException("Interrupted while initializing bk journal manager",
ie);
}
@@ -322,13 +330,14 @@ public class BookKeeperJournalManager im
} catch (KeeperException ke) {
throw new IOException("Error in zookeeper while creating ledger", ke);
} catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
throw new IOException("Interrupted creating ledger", ie);
}
try {
String znodePath = inprogressZNode(txId);
EditLogLedgerMetadata l = new EditLogLedgerMetadata(znodePath,
- HdfsConstants.LAYOUT_VERSION, currentLedger.getId(), txId);
+ HdfsConstants.LAYOUT_VERSION, currentLedger.getId(), txId);
/* Write the ledger metadata out to the inprogress ledger znode
* This can fail if for some reason our write lock has
* expired (@see WriteLock) and another process has managed to
@@ -356,6 +365,7 @@ public class BookKeeperJournalManager im
//log & ignore, an IOException will be thrown soon
LOG.error("Error closing ledger", bke);
} catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
LOG.warn("Interrupted while closing ledger", ie);
}
}
@@ -425,6 +435,7 @@ public class BookKeeperJournalManager im
} catch (KeeperException e) {
throw new IOException("Error finalising ledger", e);
} catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
throw new IOException("Error finalising ledger", ie);
}
}
@@ -454,6 +465,7 @@ public class BookKeeperJournalManager im
} catch (BKException e) {
throw new IOException("Could not open ledger for " + fromTxId, e);
} catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
throw new IOException("Interrupted opening ledger for "
+ fromTxId, ie);
}
@@ -567,6 +579,7 @@ public class BookKeeperJournalManager im
} catch (KeeperException ke) {
throw new IOException("Couldn't get list of inprogress segments", ke);
} catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
throw new IOException("Interrupted getting list of inprogress segments",
ie);
}
@@ -583,6 +596,7 @@ public class BookKeeperJournalManager im
zkc.delete(l.getZkPath(), stat.getVersion());
bkc.deleteLedger(l.getLedgerId());
} catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
LOG.error("Interrupted while purging " + l, ie);
} catch (BKException bke) {
LOG.error("Couldn't delete ledger from bookkeeper", bke);
@@ -601,6 +615,7 @@ public class BookKeeperJournalManager im
} catch (BKException bke) {
throw new IOException("Couldn't close bookkeeper client", bke);
} catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
throw new IOException("Interrupted while closing journal manager", ie);
}
}
@@ -635,6 +650,7 @@ public class BookKeeperJournalManager im
} catch (BKException bke) {
throw new IOException("Exception opening ledger for " + l, bke);
} catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
throw new IOException("Interrupted opening ledger for " + l, ie);
}
@@ -692,6 +708,7 @@ public class BookKeeperJournalManager im
} catch (KeeperException e) {
throw new IOException("Exception reading ledger list from zk", e);
} catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
throw new IOException("Interrupted getting list of ledgers from zk", ie);
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java Mon Sep 10 18:45:45 2012
@@ -29,6 +29,10 @@ import org.apache.zookeeper.KeeperExcept
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.Stat;
+import org.apache.hadoop.contrib.bkjournal.BKJournalProtos.CurrentInprogressProto;
+import com.google.protobuf.TextFormat;
+import static com.google.common.base.Charsets.UTF_8;
+
/**
* Distributed write permission lock, using ZooKeeper. Read the version number
* and return the current inprogress node path available in CurrentInprogress
@@ -42,29 +46,28 @@ import org.apache.zookeeper.data.Stat;
*/
class CurrentInprogress {
- private static final String CONTENT_DELIMITER = ",";
-
static final Log LOG = LogFactory.getLog(CurrentInprogress.class);
private final ZooKeeper zkc;
private final String currentInprogressNode;
private volatile int versionNumberForPermission = -1;
- private static final int CURRENT_INPROGRESS_LAYOUT_VERSION = -1;
private final String hostName = InetAddress.getLocalHost().toString();
CurrentInprogress(ZooKeeper zkc, String lockpath) throws IOException {
this.currentInprogressNode = lockpath;
this.zkc = zkc;
try {
- Stat isCurrentInprogressNodeExists = zkc.exists(lockpath, false);
+ Stat isCurrentInprogressNodeExists = zkc.exists(currentInprogressNode,
+ false);
if (isCurrentInprogressNodeExists == null) {
try {
- zkc.create(lockpath, null, Ids.OPEN_ACL_UNSAFE,
- CreateMode.PERSISTENT);
+ zkc.create(currentInprogressNode, null, Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
} catch (NodeExistsException e) {
// Node might created by other process at the same time. Ignore it.
if (LOG.isDebugEnabled()) {
- LOG.debug(lockpath + " already created by other process.", e);
+ LOG.debug(currentInprogressNode + " already created by other process.",
+ e);
}
}
}
@@ -83,10 +86,13 @@ class CurrentInprogress {
* @throws IOException
*/
void update(String path) throws IOException {
- String content = CURRENT_INPROGRESS_LAYOUT_VERSION
- + CONTENT_DELIMITER + hostName + CONTENT_DELIMITER + path;
+ CurrentInprogressProto.Builder builder = CurrentInprogressProto.newBuilder();
+ builder.setPath(path).setHostname(hostName);
+
+ String content = TextFormat.printToString(builder.build());
+
try {
- zkc.setData(this.currentInprogressNode, content.getBytes(),
+ zkc.setData(this.currentInprogressNode, content.getBytes(UTF_8),
this.versionNumberForPermission);
} catch (KeeperException e) {
throw new IOException("Exception when setting the data "
@@ -123,23 +129,12 @@ class CurrentInprogress {
}
this.versionNumberForPermission = stat.getVersion();
if (data != null) {
- String stringData = new String(data);
- LOG.info("Read data[layout version number,hostname,inprogressNode path]"
- + "= [" + stringData + "] from CurrentInprogress");
- String[] contents = stringData.split(CONTENT_DELIMITER);
- assert contents.length == 3 : "As per the current data format, "
- + "CurrentInprogress node data should contain 3 fields. "
- + "i.e layout version number,hostname,inprogressNode path";
- String layoutVersion = contents[0];
- if (Long.valueOf(layoutVersion) > CURRENT_INPROGRESS_LAYOUT_VERSION) {
- throw new IOException(
- "Supported layout version of CurrentInprogress node is : "
- + CURRENT_INPROGRESS_LAYOUT_VERSION
- + " . Layout version of CurrentInprogress node in ZK is : "
- + layoutVersion);
+ CurrentInprogressProto.Builder builder = CurrentInprogressProto.newBuilder();
+ TextFormat.merge(new String(data, UTF_8), builder);
+ if (!builder.isInitialized()) {
+ throw new IOException("Invalid/Incomplete data in znode");
}
- String inprogressNodePath = contents[2];
- return inprogressNodePath;
+ return builder.build().getPath();
} else {
LOG.info("No data available in CurrentInprogress");
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java Mon Sep 10 18:45:45 2012
@@ -29,6 +29,10 @@ import org.apache.hadoop.hdfs.protocol.H
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.contrib.bkjournal.BKJournalProtos.EditLogLedgerProto;
+import com.google.protobuf.TextFormat;
+import static com.google.common.base.Charsets.UTF_8;
+
/**
* Utility class for storing the metadata associated
* with a single edit log segment, stored in a single ledger
@@ -37,8 +41,8 @@ public class EditLogLedgerMetadata {
static final Log LOG = LogFactory.getLog(EditLogLedgerMetadata.class);
private String zkPath;
+ private final int dataLayoutVersion;
private final long ledgerId;
- private final int version;
private final long firstTxId;
private long lastTxId;
private boolean inprogress;
@@ -57,21 +61,22 @@ public class EditLogLedgerMetadata {
}
};
- EditLogLedgerMetadata(String zkPath, int version,
+ EditLogLedgerMetadata(String zkPath, int dataLayoutVersion,
long ledgerId, long firstTxId) {
this.zkPath = zkPath;
+ this.dataLayoutVersion = dataLayoutVersion;
this.ledgerId = ledgerId;
- this.version = version;
this.firstTxId = firstTxId;
this.lastTxId = HdfsConstants.INVALID_TXID;
this.inprogress = true;
}
- EditLogLedgerMetadata(String zkPath, int version, long ledgerId,
- long firstTxId, long lastTxId) {
+ EditLogLedgerMetadata(String zkPath, int dataLayoutVersion,
+ long ledgerId, long firstTxId,
+ long lastTxId) {
this.zkPath = zkPath;
+ this.dataLayoutVersion = dataLayoutVersion;
this.ledgerId = ledgerId;
- this.version = version;
this.firstTxId = firstTxId;
this.lastTxId = lastTxId;
this.inprogress = false;
@@ -93,14 +98,14 @@ public class EditLogLedgerMetadata {
return ledgerId;
}
- int getVersion() {
- return version;
- }
-
boolean isInProgress() {
return this.inprogress;
}
+ int getDataLayoutVersion() {
+ return this.dataLayoutVersion;
+ }
+
void finalizeLedger(long newLastTxId) {
assert this.lastTxId == HdfsConstants.INVALID_TXID;
this.lastTxId = newLastTxId;
@@ -111,22 +116,27 @@ public class EditLogLedgerMetadata {
throws IOException, KeeperException.NoNodeException {
try {
byte[] data = zkc.getData(path, false, null);
- String[] parts = new String(data).split(";");
- if (parts.length == 3) {
- int version = Integer.valueOf(parts[0]);
- long ledgerId = Long.valueOf(parts[1]);
- long txId = Long.valueOf(parts[2]);
- return new EditLogLedgerMetadata(path, version, ledgerId, txId);
- } else if (parts.length == 4) {
- int version = Integer.valueOf(parts[0]);
- long ledgerId = Long.valueOf(parts[1]);
- long firstTxId = Long.valueOf(parts[2]);
- long lastTxId = Long.valueOf(parts[3]);
- return new EditLogLedgerMetadata(path, version, ledgerId,
- firstTxId, lastTxId);
+
+ EditLogLedgerProto.Builder builder = EditLogLedgerProto.newBuilder();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Reading " + path + " data: " + new String(data, UTF_8));
+ }
+ TextFormat.merge(new String(data, UTF_8), builder);
+ if (!builder.isInitialized()) {
+ throw new IOException("Invalid/Incomplete data in znode");
+ }
+ EditLogLedgerProto ledger = builder.build();
+
+ int dataLayoutVersion = ledger.getDataLayoutVersion();
+ long ledgerId = ledger.getLedgerId();
+ long firstTxId = ledger.getFirstTxId();
+ if (ledger.hasLastTxId()) {
+ long lastTxId = ledger.getLastTxId();
+ return new EditLogLedgerMetadata(path, dataLayoutVersion,
+ ledgerId, firstTxId, lastTxId);
} else {
- throw new IOException("Invalid ledger entry, "
- + new String(data));
+ return new EditLogLedgerMetadata(path, dataLayoutVersion,
+ ledgerId, firstTxId);
}
} catch(KeeperException.NoNodeException nne) {
throw nne;
@@ -140,17 +150,17 @@ public class EditLogLedgerMetadata {
void write(ZooKeeper zkc, String path)
throws IOException, KeeperException.NodeExistsException {
this.zkPath = path;
- String finalisedData;
- if (inprogress) {
- finalisedData = String.format("%d;%d;%d",
- version, ledgerId, firstTxId);
- } else {
- finalisedData = String.format("%d;%d;%d;%d",
- version, ledgerId, firstTxId, lastTxId);
+
+ EditLogLedgerProto.Builder builder = EditLogLedgerProto.newBuilder();
+ builder.setDataLayoutVersion(dataLayoutVersion)
+ .setLedgerId(ledgerId).setFirstTxId(firstTxId);
+
+ if (!inprogress) {
+ builder.setLastTxId(lastTxId);
}
try {
- zkc.create(path, finalisedData.getBytes(), Ids.OPEN_ACL_UNSAFE,
- CreateMode.PERSISTENT);
+ zkc.create(path, TextFormat.printToString(builder.build()).getBytes(UTF_8),
+ Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
} catch (KeeperException.NodeExistsException nee) {
throw nee;
} catch (KeeperException e) {
@@ -183,9 +193,9 @@ public class EditLogLedgerMetadata {
}
EditLogLedgerMetadata ol = (EditLogLedgerMetadata)o;
return ledgerId == ol.ledgerId
+ && dataLayoutVersion == ol.dataLayoutVersion
&& firstTxId == ol.firstTxId
- && lastTxId == ol.lastTxId
- && version == ol.version;
+ && lastTxId == ol.lastTxId;
}
public int hashCode() {
@@ -193,15 +203,15 @@ public class EditLogLedgerMetadata {
hash = hash * 31 + (int) ledgerId;
hash = hash * 31 + (int) firstTxId;
hash = hash * 31 + (int) lastTxId;
- hash = hash * 31 + (int) version;
+ hash = hash * 31 + (int) dataLayoutVersion;
return hash;
}
public String toString() {
return "[LedgerId:"+ledgerId +
", firstTxId:" + firstTxId +
- ", lastTxId:" + lastTxId +
- ", version:" + version + "]";
+ ", lastTxId:" + lastTxId +
+ ", dataLayoutVersion:" + dataLayoutVersion + "]";
}
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java Mon Sep 10 18:45:45 2012
@@ -27,6 +27,10 @@ import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.Stat;
+import org.apache.hadoop.contrib.bkjournal.BKJournalProtos.MaxTxIdProto;
+import com.google.protobuf.TextFormat;
+import static com.google.common.base.Charsets.UTF_8;
+
/**
* Utility class for storing and reading
* the max seen txid in zookeeper
@@ -55,14 +59,16 @@ class MaxTxId {
}
synchronized void reset(long maxTxId) throws IOException {
- String txidStr = Long.toString(maxTxId);
try {
+ MaxTxIdProto.Builder builder = MaxTxIdProto.newBuilder().setTxId(maxTxId);
+
+ byte[] data = TextFormat.printToString(builder.build()).getBytes(UTF_8);
if (currentStat != null) {
- currentStat = zkc.setData(path, txidStr.getBytes("UTF-8"), currentStat
+ currentStat = zkc.setData(path, data, currentStat
.getVersion());
} else {
- zkc.create(path, txidStr.getBytes("UTF-8"), Ids.OPEN_ACL_UNSAFE,
- CreateMode.PERSISTENT);
+ zkc.create(path, data, Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
}
} catch (KeeperException e) {
throw new IOException("Error writing max tx id", e);
@@ -77,9 +83,16 @@ class MaxTxId {
if (currentStat == null) {
return 0;
} else {
+
byte[] bytes = zkc.getData(path, false, currentStat);
- String txidString = new String(bytes, "UTF-8");
- return Long.valueOf(txidString);
+
+ MaxTxIdProto.Builder builder = MaxTxIdProto.newBuilder();
+ TextFormat.merge(new String(bytes, UTF_8), builder);
+ if (!builder.isInitialized()) {
+ throw new IOException("Invalid/Incomplete data in znode");
+ }
+
+ return builder.build().getTxId();
}
} catch (KeeperException e) {
throw new IOException("Error reading the max tx id from zk", e);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java Mon Sep 10 18:45:45 2012
@@ -23,6 +23,7 @@ import java.net.InetSocketAddress;
import java.net.URI;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import java.util.Random;
import org.apache.bookkeeper.util.LocalBookKeeper;
import org.apache.commons.logging.Log;
@@ -42,6 +43,8 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+
public class TestBookKeeperConfiguration {
private static final Log LOG = LogFactory
.getLog(TestBookKeeperConfiguration.class);
@@ -73,6 +76,11 @@ public class TestBookKeeperConfiguration
return zkc;
}
+ private NamespaceInfo newNSInfo() {
+ Random r = new Random();
+ return new NamespaceInfo(r.nextInt(), "testCluster", "TestBPID", -1);
+ }
+
@BeforeClass
public static void setupZooKeeper() throws Exception {
// create a ZooKeeper server(dataDir, dataLogDir, port)
@@ -137,8 +145,10 @@ public class TestBookKeeperConfiguration
bkAvailablePath);
Assert.assertNull(bkAvailablePath + " already exists", zkc.exists(
bkAvailablePath, false));
- bkjm = new BookKeeperJournalManager(conf, URI.create("bookkeeper://"
- + HOSTPORT + "/hdfsjournal-WithBKPath"));
+ NamespaceInfo nsi = newNSInfo();
+ bkjm = new BookKeeperJournalManager(conf,
+ URI.create("bookkeeper://" + HOSTPORT + "/hdfsjournal-WithBKPath"),
+ nsi);
Assert.assertNotNull("Bookie available path : " + bkAvailablePath
+ " doesn't exists", zkc.exists(bkAvailablePath, false));
}
@@ -152,8 +162,10 @@ public class TestBookKeeperConfiguration
Configuration conf = new Configuration();
Assert.assertNull(BK_ROOT_PATH + " already exists", zkc.exists(
BK_ROOT_PATH, false));
- new BookKeeperJournalManager(conf, URI.create("bookkeeper://" + HOSTPORT
- + "/hdfsjournal-DefaultBKPath"));
+ NamespaceInfo nsi = newNSInfo();
+ bkjm = new BookKeeperJournalManager(conf,
+ URI.create("bookkeeper://" + HOSTPORT + "/hdfsjournal-DefaultBKPath"),
+ nsi);
Assert.assertNotNull("Bookie available path : " + BK_ROOT_PATH
+ " doesn't exists", zkc.exists(BK_ROOT_PATH, false));
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java Mon Sep 10 18:45:45 2012
@@ -29,6 +29,7 @@ import org.mockito.Mockito;
import java.io.IOException;
import java.net.URI;
import java.util.List;
+import java.util.Random;
import org.apache.hadoop.conf.Configuration;
@@ -37,6 +38,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogTestUtil;
import org.apache.hadoop.hdfs.server.namenode.JournalManager;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.bookkeeper.proto.BookieServer;
import org.apache.zookeeper.CreateMode;
@@ -78,10 +80,17 @@ public class TestBookKeeperJournalManage
zkc.close();
}
+ private NamespaceInfo newNSInfo() {
+ Random r = new Random();
+ return new NamespaceInfo(r.nextInt(), "testCluster", "TestBPID", -1);
+ }
+
@Test
public void testSimpleWrite() throws Exception {
+ NamespaceInfo nsi = newNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
- BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"));
+ BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi);
+
EditLogOutputStream out = bkjm.startLogSegment(1);
for (long i = 1 ; i <= 100; i++) {
FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
@@ -99,8 +108,10 @@ public class TestBookKeeperJournalManage
@Test
public void testNumberOfTransactions() throws Exception {
+ NamespaceInfo nsi = newNSInfo();
+
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
- BKJMUtil.createJournalURI("/hdfsjournal-txncount"));
+ BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi);
EditLogOutputStream out = bkjm.startLogSegment(1);
for (long i = 1 ; i <= 100; i++) {
FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
@@ -116,8 +127,10 @@ public class TestBookKeeperJournalManage
@Test
public void testNumberOfTransactionsWithGaps() throws Exception {
+ NamespaceInfo nsi = newNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
- BKJMUtil.createJournalURI("/hdfsjournal-gaps"));
+ BKJMUtil.createJournalURI("/hdfsjournal-gaps"), nsi);
+
long txid = 1;
for (long i = 0; i < 3; i++) {
long start = txid;
@@ -151,8 +164,10 @@ public class TestBookKeeperJournalManage
@Test
public void testNumberOfTransactionsWithInprogressAtEnd() throws Exception {
+ NamespaceInfo nsi = newNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
- BKJMUtil.createJournalURI("/hdfsjournal-inprogressAtEnd"));
+ BKJMUtil.createJournalURI("/hdfsjournal-inprogressAtEnd"), nsi);
+
long txid = 1;
for (long i = 0; i < 3; i++) {
long start = txid;
@@ -190,8 +205,10 @@ public class TestBookKeeperJournalManage
*/
@Test
public void testWriteRestartFrom1() throws Exception {
+ NamespaceInfo nsi = newNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
- BKJMUtil.createJournalURI("/hdfsjournal-restartFrom1"));
+ BKJMUtil.createJournalURI("/hdfsjournal-restartFrom1"), nsi);
+
long txid = 1;
long start = txid;
EditLogOutputStream out = bkjm.startLogSegment(txid);
@@ -245,11 +262,15 @@ public class TestBookKeeperJournalManage
@Test
public void testTwoWriters() throws Exception {
long start = 1;
+ NamespaceInfo nsi = newNSInfo();
+
BookKeeperJournalManager bkjm1 = new BookKeeperJournalManager(conf,
- BKJMUtil.createJournalURI("/hdfsjournal-dualWriter"));
+ BKJMUtil.createJournalURI("/hdfsjournal-dualWriter"), nsi);
+
BookKeeperJournalManager bkjm2 = new BookKeeperJournalManager(conf,
- BKJMUtil.createJournalURI("/hdfsjournal-dualWriter"));
-
+ BKJMUtil.createJournalURI("/hdfsjournal-dualWriter"), nsi);
+
+
EditLogOutputStream out1 = bkjm1.startLogSegment(start);
try {
bkjm2.startLogSegment(start);
@@ -263,8 +284,11 @@ public class TestBookKeeperJournalManage
@Test
public void testSimpleRead() throws Exception {
+ NamespaceInfo nsi = newNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
- BKJMUtil.createJournalURI("/hdfsjournal-simpleread"));
+ BKJMUtil.createJournalURI("/hdfsjournal-simpleread"),
+ nsi);
+
final long numTransactions = 10000;
EditLogOutputStream out = bkjm.startLogSegment(1);
for (long i = 1 ; i <= numTransactions; i++) {
@@ -287,8 +311,11 @@ public class TestBookKeeperJournalManage
@Test
public void testSimpleRecovery() throws Exception {
+ NamespaceInfo nsi = newNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
- BKJMUtil.createJournalURI("/hdfsjournal-simplerecovery"));
+ BKJMUtil.createJournalURI("/hdfsjournal-simplerecovery"),
+ nsi);
+
EditLogOutputStream out = bkjm.startLogSegment(1);
for (long i = 1 ; i <= 100; i++) {
FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
@@ -334,8 +361,10 @@ public class TestBookKeeperJournalManage
conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE,
ensembleSize);
long txid = 1;
+ NamespaceInfo nsi = newNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
- BKJMUtil.createJournalURI("/hdfsjournal-allbookiefailure"));
+ BKJMUtil.createJournalURI("/hdfsjournal-allbookiefailure"),
+ nsi);
EditLogOutputStream out = bkjm.startLogSegment(txid);
for (long i = 1 ; i <= 3; i++) {
@@ -416,8 +445,12 @@ public class TestBookKeeperJournalManage
conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE,
ensembleSize);
long txid = 1;
+
+ NamespaceInfo nsi = newNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
- BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure"));
+ BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure"),
+ nsi);
+
EditLogOutputStream out = bkjm.startLogSegment(txid);
for (long i = 1 ; i <= 3; i++) {
FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
@@ -464,7 +497,9 @@ public class TestBookKeeperJournalManage
@Test
public void testEmptyInprogressNode() throws Exception {
URI uri = BKJMUtil.createJournalURI("/hdfsjournal-emptyInprogress");
- BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri);
+ NamespaceInfo nsi = newNSInfo();
+ BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri,
+ nsi);
EditLogOutputStream out = bkjm.startLogSegment(1);
for (long i = 1; i <= 100; i++) {
@@ -481,7 +516,7 @@ public class TestBookKeeperJournalManage
String inprogressZNode = bkjm.inprogressZNode(101);
zkc.setData(inprogressZNode, new byte[0], -1);
- bkjm = new BookKeeperJournalManager(conf, uri);
+ bkjm = new BookKeeperJournalManager(conf, uri, nsi);
try {
bkjm.recoverUnfinalizedSegments();
fail("Should have failed. There should be no way of creating"
@@ -489,7 +524,7 @@ public class TestBookKeeperJournalManage
} catch (IOException e) {
// correct behaviour
assertTrue("Exception different than expected", e.getMessage().contains(
- "Invalid ledger entry,"));
+ "Invalid/Incomplete data in znode"));
} finally {
bkjm.close();
}
@@ -503,7 +538,9 @@ public class TestBookKeeperJournalManage
@Test
public void testCorruptInprogressNode() throws Exception {
URI uri = BKJMUtil.createJournalURI("/hdfsjournal-corruptInprogress");
- BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri);
+ NamespaceInfo nsi = newNSInfo();
+ BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri,
+ nsi);
EditLogOutputStream out = bkjm.startLogSegment(1);
for (long i = 1; i <= 100; i++) {
@@ -521,7 +558,7 @@ public class TestBookKeeperJournalManage
String inprogressZNode = bkjm.inprogressZNode(101);
zkc.setData(inprogressZNode, "WholeLottaJunk".getBytes(), -1);
- bkjm = new BookKeeperJournalManager(conf, uri);
+ bkjm = new BookKeeperJournalManager(conf, uri, nsi);
try {
bkjm.recoverUnfinalizedSegments();
fail("Should have failed. There should be no way of creating"
@@ -529,8 +566,7 @@ public class TestBookKeeperJournalManage
} catch (IOException e) {
// correct behaviour
assertTrue("Exception different than expected", e.getMessage().contains(
- "Invalid ledger entry,"));
-
+ "has no field named"));
} finally {
bkjm.close();
}
@@ -544,7 +580,9 @@ public class TestBookKeeperJournalManage
@Test
public void testEmptyInprogressLedger() throws Exception {
URI uri = BKJMUtil.createJournalURI("/hdfsjournal-emptyInprogressLedger");
- BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri);
+ NamespaceInfo nsi = newNSInfo();
+ BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri,
+ nsi);
EditLogOutputStream out = bkjm.startLogSegment(1);
for (long i = 1; i <= 100; i++) {
@@ -559,7 +597,7 @@ public class TestBookKeeperJournalManage
out.close();
bkjm.close();
- bkjm = new BookKeeperJournalManager(conf, uri);
+ bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.recoverUnfinalizedSegments();
out = bkjm.startLogSegment(101);
for (long i = 1; i <= 100; i++) {
@@ -581,7 +619,9 @@ public class TestBookKeeperJournalManage
public void testRefinalizeAlreadyFinalizedInprogress() throws Exception {
URI uri = BKJMUtil
.createJournalURI("/hdfsjournal-refinalizeInprogressLedger");
- BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri);
+ NamespaceInfo nsi = newNSInfo();
+ BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri,
+ nsi);
EditLogOutputStream out = bkjm.startLogSegment(1);
for (long i = 1; i <= 100; i++) {
@@ -601,7 +641,7 @@ public class TestBookKeeperJournalManage
byte[] inprogressData = zkc.getData(inprogressZNode, false, null);
// finalize
- bkjm = new BookKeeperJournalManager(conf, uri);
+ bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.recoverUnfinalizedSegments();
bkjm.close();
@@ -613,7 +653,7 @@ public class TestBookKeeperJournalManage
CreateMode.PERSISTENT);
// should work fine
- bkjm = new BookKeeperJournalManager(conf, uri);
+ bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.recoverUnfinalizedSegments();
bkjm.close();
}
@@ -626,7 +666,10 @@ public class TestBookKeeperJournalManage
@Test
public void testEditLogFileNotExistsWhenReadingMetadata() throws Exception {
URI uri = BKJMUtil.createJournalURI("/hdfsjournal-editlogfile");
- BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri);
+ NamespaceInfo nsi = newNSInfo();
+ BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri,
+ nsi);
+
try {
// start new inprogress log segment with txid=1
// and write transactions till txid=50
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1380987-1383029
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java Mon Sep 10 18:45:45 2012
@@ -321,7 +321,7 @@ public class Hdfs extends AbstractFileSy
@Override
public void mkdir(Path dir, FsPermission permission, boolean createParent)
throws IOException, UnresolvedLinkException {
- dfs.mkdirs(getUriPath(dir), permission, createParent);
+ dfs.primitiveMkdir(getUriPath(dir), permission, createParent);
}
@SuppressWarnings("deprecation")
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Mon Sep 10 18:45:45 2012
@@ -1956,34 +1956,29 @@ public class DFSClient implements java.i
*/
public boolean mkdirs(String src, FsPermission permission,
boolean createParent) throws IOException {
- checkOpen();
if (permission == null) {
permission = FsPermission.getDefault();
}
FsPermission masked = permission.applyUMask(dfsClientConf.uMask);
- if(LOG.isDebugEnabled()) {
- LOG.debug(src + ": masked=" + masked);
- }
- try {
- return namenode.mkdirs(src, masked, createParent);
- } catch(RemoteException re) {
- throw re.unwrapRemoteException(AccessControlException.class,
- InvalidPathException.class,
- FileAlreadyExistsException.class,
- FileNotFoundException.class,
- ParentNotDirectoryException.class,
- SafeModeException.class,
- NSQuotaExceededException.class,
- UnresolvedPathException.class);
- }
+ return primitiveMkdir(src, masked, createParent);
}
-
+
/**
* Same {{@link #mkdirs(String, FsPermission, boolean)} except
* that the permissions has already been masked against umask.
*/
public boolean primitiveMkdir(String src, FsPermission absPermission)
throws IOException {
+ return primitiveMkdir(src, absPermission, true);
+ }
+
+ /**
+ * Same {{@link #mkdirs(String, FsPermission, boolean)} except
+ * that the permissions has already been masked against umask.
+ */
+ public boolean primitiveMkdir(String src, FsPermission absPermission,
+ boolean createParent)
+ throws IOException {
checkOpen();
if (absPermission == null) {
absPermission =
@@ -1994,15 +1989,20 @@ public class DFSClient implements java.i
LOG.debug(src + ": masked=" + absPermission);
}
try {
- return namenode.mkdirs(src, absPermission, true);
+ return namenode.mkdirs(src, absPermission, createParent);
} catch(RemoteException re) {
throw re.unwrapRemoteException(AccessControlException.class,
+ InvalidPathException.class,
+ FileAlreadyExistsException.class,
+ FileNotFoundException.class,
+ ParentNotDirectoryException.class,
+ SafeModeException.class,
NSQuotaExceededException.class,
DSQuotaExceededException.class,
UnresolvedPathException.class);
}
}
-
+
/**
* Get {@link ContentSummary} rooted at the specified directory.
* @param path The string representation of the path
@@ -2074,10 +2074,7 @@ public class DFSClient implements java.i
}
boolean shouldTryShortCircuitRead(InetSocketAddress targetAddr) {
- if (shortCircuitLocalReads && isLocalAddress(targetAddr)) {
- return true;
- }
- return false;
+ return shortCircuitLocalReads && isLocalAddress(targetAddr);
}
void reportChecksumFailure(String file, ExtendedBlock blk, DatanodeInfo dn) {
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Mon Sep 10 18:45:45 2012
@@ -329,6 +329,10 @@ public class DFSConfigKeys extends Commo
"dfs.image.transfer.bandwidthPerSec";
public static final long DFS_IMAGE_TRANSFER_RATE_DEFAULT = 0; //no throttling
+ // Image transfer timeout
+ public static final String DFS_IMAGE_TRANSFER_TIMEOUT_KEY = "dfs.image.transfer.timeout";
+ public static final int DFS_IMAGE_TRANSFER_TIMEOUT_DEFAULT = 60 * 1000;
+
//Keys with no defaults
public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins";
public static final String DFS_DATANODE_FSDATASET_FACTORY_KEY = "dfs.datanode.fsdataset.factory";
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java Mon Sep 10 18:45:45 2012
@@ -243,6 +243,10 @@ public class DFSInputStream extends FSIn
locatedBlocks.getFileLength() + lastBlockBeingWrittenLength;
}
+ private synchronized boolean blockUnderConstruction() {
+ return locatedBlocks.isUnderConstruction();
+ }
+
/**
* Returns the datanode from which the stream is currently reading.
*/
@@ -878,7 +882,9 @@ public class DFSInputStream extends FSIn
String clientName)
throws IOException {
- if (dfsClient.shouldTryShortCircuitRead(dnAddr)) {
+ // Can't local read a block under construction, see HDFS-2757
+ if (dfsClient.shouldTryShortCircuitRead(dnAddr) &&
+ !blockUnderConstruction()) {
return DFSClient.getLocalBlockReader(dfsClient.conf, src, block,
blockToken, chosenNode, dfsClient.hdfsTimeout, startOffset,
dfsClient.connectToDnViaHostname());
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java Mon Sep 10 18:45:45 2012
@@ -51,6 +51,8 @@ import org.apache.hadoop.hdfs.util.DataT
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Time;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Scans the block files under a block pool and verifies that the
* files are not corrupt.
@@ -255,6 +257,11 @@ class BlockPoolSliceScanner {
}
}
+ @VisibleForTesting
+ long getTotalScans() {
+ return totalScans;
+ }
+
/** @return the last scan time for the block pool. */
long getLastScanTime() {
return lastScanTime.get();
@@ -563,7 +570,24 @@ class BlockPoolSliceScanner {
currentPeriodStart = Time.now();
}
+ private synchronized boolean workRemainingInCurrentPeriod() {
+ if (bytesLeft <= 0 && Time.now() < currentPeriodStart + scanPeriod) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skipping scan since bytesLeft=" + bytesLeft + ", Start=" +
+ currentPeriodStart + ", period=" + scanPeriod + ", now=" +
+ Time.now() + " " + blockPoolId);
+ }
+ return false;
+ } else {
+ return true;
+ }
+ }
+
void scanBlockPoolSlice() {
+ if (!workRemainingInCurrentPeriod()) {
+ return;
+ }
+
// Create a new processedBlocks structure
processedBlocks = new HashMap<Long, Integer>();
if (!assignInitialVerificationTimes()) {
@@ -608,14 +632,14 @@ class BlockPoolSliceScanner {
LOG.warn("RuntimeException during BlockPoolScanner.scan()", e);
throw e;
} finally {
- cleanUp();
+ rollVerificationLogs();
if (LOG.isDebugEnabled()) {
LOG.debug("Done scanning block pool: " + blockPoolId);
}
}
}
- private synchronized void cleanUp() {
+ private synchronized void rollVerificationLogs() {
if (verificationLog != null) {
try {
verificationLog.logs.roll();
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java Mon Sep 10 18:45:45 2012
@@ -34,6 +34,8 @@ import org.apache.hadoop.hdfs.protocol.E
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* DataBlockScanner manages block scanning for all the block pools. For each
* block pool a {@link BlockPoolSliceScanner} is created which runs in a separate
@@ -47,6 +49,8 @@ public class DataBlockScanner implements
private final FsDatasetSpi<? extends FsVolumeSpi> dataset;
private final Configuration conf;
+ static final int SLEEP_PERIOD_MS = 5 * 1000;
+
/**
* Map to find the BlockPoolScanner for a given block pool id. This is updated
* when a BPOfferService becomes alive or dies.
@@ -68,10 +72,10 @@ public class DataBlockScanner implements
String currentBpId = "";
boolean firstRun = true;
while (datanode.shouldRun && !Thread.interrupted()) {
- //Sleep everytime except in the first interation.
+ //Sleep everytime except in the first iteration.
if (!firstRun) {
try {
- Thread.sleep(5000);
+ Thread.sleep(SLEEP_PERIOD_MS);
} catch (InterruptedException ex) {
// Interrupt itself again to set the interrupt status
blockScannerThread.interrupt();
@@ -103,7 +107,7 @@ public class DataBlockScanner implements
while ((getBlockPoolSetSize() < datanode.getAllBpOs().length)
|| (getBlockPoolSetSize() < 1)) {
try {
- Thread.sleep(5000);
+ Thread.sleep(SLEEP_PERIOD_MS);
} catch (InterruptedException e) {
blockScannerThread.interrupt();
return;
@@ -249,7 +253,7 @@ public class DataBlockScanner implements
LOG.info("Removed bpid="+blockPoolId+" from blockPoolScannerMap");
}
- // This method is used for testing
+ @VisibleForTesting
long getBlocksScannedInLastRun(String bpid) throws IOException {
BlockPoolSliceScanner bpScanner = getBPScanner(bpid);
if (bpScanner == null) {
@@ -259,6 +263,16 @@ public class DataBlockScanner implements
}
}
+ @VisibleForTesting
+ long getTotalScans(String bpid) throws IOException {
+ BlockPoolSliceScanner bpScanner = getBPScanner(bpid);
+ if (bpScanner == null) {
+ throw new IOException("Block Pool: "+bpid+" is not running");
+ } else {
+ return bpScanner.getTotalScans();
+ }
+ }
+
public void start() {
blockScannerThread = new Thread(this);
blockScannerThread.setDaemon(true);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java Mon Sep 10 18:45:45 2012
@@ -32,9 +32,12 @@ import javax.servlet.http.HttpServletRes
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.Time;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.common.StorageErrorReporter;
import org.apache.hadoop.hdfs.server.common.Storage;
@@ -43,6 +46,7 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
import org.apache.hadoop.io.MD5Hash;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
@@ -54,6 +58,8 @@ public class TransferFsImage {
public final static String CONTENT_LENGTH = "Content-Length";
public final static String MD5_HEADER = "X-MD5-Digest";
+ @VisibleForTesting
+ static int timeout = 0;
private static final Log LOG = LogFactory.getLog(TransferFsImage.class);
@@ -222,6 +228,18 @@ public class TransferFsImage {
HttpURLConnection connection = (HttpURLConnection)
SecurityUtil.openSecureHttpConnection(url);
+ if (timeout <= 0) {
+ // Set the ping interval as timeout
+ Configuration conf = new HdfsConfiguration();
+ timeout = conf.getInt(DFSConfigKeys.DFS_IMAGE_TRANSFER_TIMEOUT_KEY,
+ DFSConfigKeys.DFS_IMAGE_TRANSFER_TIMEOUT_DEFAULT);
+ }
+
+ if (timeout > 0) {
+ connection.setConnectTimeout(timeout);
+ connection.setReadTimeout(timeout);
+ }
+
if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
throw new HttpGetFailedException(
"Image transfer servlet at " + url +
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1380987-1383029
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1380987-1383029
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1380987-1383029
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1380987-1383029
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1380987-1383029
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java Mon Sep 10 18:45:45 2012
@@ -57,6 +57,8 @@ import org.apache.hadoop.util.StringUtil
import org.apache.hadoop.util.ToolRunner;
import org.junit.Test;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
+
/**
* This class tests commands from DFSShell.
*/
@@ -1480,4 +1482,95 @@ public class TestDFSShell {
}
+ /**
+ * Delete a file optionally configuring trash on the server and client.
+ */
+ private void deleteFileUsingTrash(
+ boolean serverTrash, boolean clientTrash) throws Exception {
+ // Run a cluster, optionally with trash enabled on the server
+ Configuration serverConf = new HdfsConfiguration();
+ if (serverTrash) {
+ serverConf.setLong(FS_TRASH_INTERVAL_KEY, 1);
+ }
+
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(serverConf)
+ .numDataNodes(1).format(true).build();
+ Configuration clientConf = new Configuration(serverConf);
+
+ // Create a client, optionally with trash enabled
+ if (clientTrash) {
+ clientConf.setLong(FS_TRASH_INTERVAL_KEY, 1);
+ } else {
+ clientConf.setLong(FS_TRASH_INTERVAL_KEY, 0);
+ }
+
+ FsShell shell = new FsShell(clientConf);
+ FileSystem fs = null;
+
+ try {
+ // Create and delete a file
+ fs = cluster.getFileSystem();
+ writeFile(fs, new Path(TEST_ROOT_DIR, "foo"));
+ final String testFile = TEST_ROOT_DIR + "/foo";
+ final String trashFile = shell.getCurrentTrashDir() + "/" + testFile;
+ String[] argv = new String[] { "-rm", testFile };
+ int res = ToolRunner.run(shell, argv);
+ assertEquals("rm failed", 0, res);
+
+ if (serverTrash) {
+ // If the server config was set we should use it unconditionally
+ assertTrue("File not in trash", fs.exists(new Path(trashFile)));
+ } else if (clientTrash) {
+ // If the server config was not set but the client config was
+ // set then we should use it
+ assertTrue("File not in trashed", fs.exists(new Path(trashFile)));
+ } else {
+ // If neither was set then we should not have trashed the file
+ assertFalse("File was not removed", fs.exists(new Path(testFile)));
+ assertFalse("File was trashed", fs.exists(new Path(trashFile)));
+ }
+ } finally {
+ if (fs != null) {
+ fs.close();
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ /**
+ * Test that the server trash configuration is respected when
+ * the client configuration is not set.
+ */
+ @Test
+ public void testServerConfigRespected() throws Exception {
+ deleteFileUsingTrash(true, false);
+ }
+
+ /**
+ * Test that server trash configuration is respected even when the
+ * client configuration is set.
+ */
+ @Test
+ public void testServerConfigRespectedWithClient() throws Exception {
+ deleteFileUsingTrash(true, true);
+ }
+
+ /**
+ * Test that the client trash configuration is respected when
+ * the server configuration is not set.
+ */
+ @Test
+ public void testClientConfigRespected() throws Exception {
+ deleteFileUsingTrash(false, true);
+ }
+
+ /**
+ * Test that trash is disabled by default.
+ */
+ @Test
+ public void testNoTrashConfig() throws Exception {
+ deleteFileUsingTrash(false, false);
+ }
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSTrash.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSTrash.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSTrash.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSTrash.java Mon Sep 10 18:45:45 2012
@@ -23,11 +23,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.TestTrash;
-import org.apache.hadoop.fs.Trash;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -62,53 +57,4 @@ public class TestHDFSTrash {
conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, fs.getUri().toString());
TestTrash.trashNonDefaultFS(conf);
}
-
- /** Clients should always use trash if enabled server side */
- @Test
- public void testTrashEnabledServerSide() throws IOException {
- Configuration serverConf = new HdfsConfiguration();
- Configuration clientConf = new Configuration();
-
- // Enable trash on the server and client
- serverConf.setLong(FS_TRASH_INTERVAL_KEY, 1);
- clientConf.setLong(FS_TRASH_INTERVAL_KEY, 1);
-
- MiniDFSCluster cluster2 = null;
- try {
- cluster2 = new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build();
- FileSystem fs = cluster2.getFileSystem();
- assertTrue(new Trash(fs, clientConf).isEnabled());
-
- // Disabling trash on the client is ignored
- clientConf.setLong(FS_TRASH_INTERVAL_KEY, 0);
- assertTrue(new Trash(fs, clientConf).isEnabled());
- } finally {
- if (cluster2 != null) cluster2.shutdown();
- }
- }
-
- /** Clients should always use trash if enabled client side */
- @Test
- public void testTrashEnabledClientSide() throws IOException {
- Configuration serverConf = new HdfsConfiguration();
- Configuration clientConf = new Configuration();
-
- // Disable server side
- serverConf.setLong(FS_TRASH_INTERVAL_KEY, 0);
-
- MiniDFSCluster cluster2 = null;
- try {
- cluster2 = new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build();
-
- // Client side is disabled by default
- FileSystem fs = cluster2.getFileSystem();
- assertFalse(new Trash(fs, clientConf).isEnabled());
-
- // Enabling on the client works even though its disabled on the server
- clientConf.setLong(FS_TRASH_INTERVAL_KEY, 1);
- assertTrue(new Trash(fs, clientConf).isEnabled());
- } finally {
- if (cluster2 != null) cluster2.shutdown();
- }
- }
}