You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2016/10/08 00:20:51 UTC
[08/50] [abbrv] hadoop git commit: HDFS-10957. Retire BKJM from trunk
(Vinayakumar B)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31195488/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestCurrentInprogress.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestCurrentInprogress.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestCurrentInprogress.java
deleted file mode 100644
index 169a8a8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestCurrentInprogress.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.contrib.bkjournal;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.bookkeeper.util.LocalBookKeeper;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.server.NIOServerCnxnFactory;
-import org.apache.zookeeper.server.ZooKeeperServer;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Tests that read, update, clear api from CurrentInprogress
- */
-public class TestCurrentInprogress {
- private static final Log LOG = LogFactory.getLog(TestCurrentInprogress.class);
- private static final String CURRENT_NODE_PATH = "/test";
- private static final String HOSTPORT = "127.0.0.1:2181";
- private static final int CONNECTION_TIMEOUT = 30000;
- private static NIOServerCnxnFactory serverFactory;
- private static ZooKeeperServer zks;
- private static ZooKeeper zkc;
- private static int ZooKeeperDefaultPort = 2181;
- private static File zkTmpDir;
-
- private static ZooKeeper connectZooKeeper(String ensemble)
- throws IOException, KeeperException, InterruptedException {
- final CountDownLatch latch = new CountDownLatch(1);
-
- ZooKeeper zkc = new ZooKeeper(HOSTPORT, 3600, new Watcher() {
- public void process(WatchedEvent event) {
- if (event.getState() == Watcher.Event.KeeperState.SyncConnected) {
- latch.countDown();
- }
- }
- });
- if (!latch.await(10, TimeUnit.SECONDS)) {
- throw new IOException("Zookeeper took too long to connect");
- }
- return zkc;
- }
-
- @BeforeClass
- public static void setupZooKeeper() throws Exception {
- LOG.info("Starting ZK server");
- zkTmpDir = File.createTempFile("zookeeper", "test");
- zkTmpDir.delete();
- zkTmpDir.mkdir();
- try {
- zks = new ZooKeeperServer(zkTmpDir, zkTmpDir, ZooKeeperDefaultPort);
- serverFactory = new NIOServerCnxnFactory();
- serverFactory.configure(new InetSocketAddress(ZooKeeperDefaultPort), 10);
- serverFactory.startup(zks);
- } catch (Exception e) {
- LOG.error("Exception while instantiating ZooKeeper", e);
- }
- boolean b = LocalBookKeeper.waitForServerUp(HOSTPORT, CONNECTION_TIMEOUT);
- LOG.debug("ZooKeeper server up: " + b);
- }
-
- @AfterClass
- public static void shutDownServer() {
- if (null != zks) {
- zks.shutdown();
- }
- zkTmpDir.delete();
- }
-
- @Before
- public void setup() throws Exception {
- zkc = connectZooKeeper(HOSTPORT);
- }
-
- @After
- public void teardown() throws Exception {
- if (null != zkc) {
- zkc.close();
- }
-
- }
-
- /**
- * Tests that read should be able to read the data which updated with update
- * api
- */
- @Test
- public void testReadShouldReturnTheZnodePathAfterUpdate() throws Exception {
- String data = "inprogressNode";
- CurrentInprogress ci = new CurrentInprogress(zkc, CURRENT_NODE_PATH);
- ci.init();
- ci.update(data);
- String inprogressNodePath = ci.read();
- assertEquals("Not returning inprogressZnode", "inprogressNode",
- inprogressNodePath);
- }
-
- /**
- * Tests that read should return null if we clear the updated data in
- * CurrentInprogress node
- */
- @Test
- public void testReadShouldReturnNullAfterClear() throws Exception {
- CurrentInprogress ci = new CurrentInprogress(zkc, CURRENT_NODE_PATH);
- ci.init();
- ci.update("myInprogressZnode");
- ci.read();
- ci.clear();
- String inprogressNodePath = ci.read();
- assertEquals("Expecting null to be return", null, inprogressNodePath);
- }
-
- /**
- * Tests that update should throw IOE, if version number modifies between read
- * and update
- */
- @Test(expected = IOException.class)
- public void testUpdateShouldFailWithIOEIfVersionNumberChangedAfterRead()
- throws Exception {
- CurrentInprogress ci = new CurrentInprogress(zkc, CURRENT_NODE_PATH);
- ci.init();
- ci.update("myInprogressZnode");
- assertEquals("Not returning myInprogressZnode", "myInprogressZnode", ci
- .read());
- // Updating data in-between to change the data to change the version number
- ci.update("YourInprogressZnode");
- ci.update("myInprogressZnode");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31195488/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java
deleted file mode 100644
index 7a7af06..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.namenode;
-
-import java.io.IOException;
-import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.OpInstanceCache;
-
-/**
- * Utilities for testing edit logs
- */
-public class FSEditLogTestUtil {
- private static OpInstanceCache cache = new OpInstanceCache();
-
- public static FSEditLogOp getNoOpInstance() {
- return FSEditLogOp.LogSegmentOp.getInstance(cache,
- FSEditLogOpCodes.OP_END_LOG_SEGMENT);
- }
-
- public static long countTransactionsInStream(EditLogInputStream in)
- throws IOException {
- FSEditLogLoader.EditLogValidation validation =
- FSEditLogLoader.scanEditLog(in, Long.MAX_VALUE);
- return (validation.getEndTxId() - in.getFirstTxId()) + 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31195488/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/resources/log4j.properties b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/resources/log4j.properties
deleted file mode 100644
index 52aac43..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,55 +0,0 @@
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-
-#
-# Bookkeeper Journal Logging Configuration
-#
-
-# Format is "<default threshold> (, <appender>)+
-
-# DEFAULT: console appender only
-log4j.rootLogger=DEBUG, CONSOLE
-
-# Example with rolling log file
-#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE
-
-# Example with rolling log file and tracing
-#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE
-
-#
-# Log INFO level and above messages to the console
-#
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.Threshold=INFO
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n
-
-#
-# Add ROLLINGFILE to rootLogger to get log file output
-# Log DEBUG level and above messages to a log file
-log4j.appender.ROLLINGFILE=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.ROLLINGFILE.Threshold=DEBUG
-log4j.appender.ROLLINGFILE.File=hdfs-namenode.log
-log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout
-log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n
-
-# Max log file size of 10MB
-log4j.appender.ROLLINGFILE.MaxFileSize=10MB
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31195488/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
index b743233..5625244 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
@@ -38,7 +38,6 @@ HDFS High Availability
* [Securing access to ZooKeeper](#Securing_access_to_ZooKeeper)
* [Verifying automatic failover](#Verifying_automatic_failover)
* [Automatic Failover FAQ](#Automatic_Failover_FAQ)
- * [BookKeeper as a Shared storage (EXPERIMENTAL)](#BookKeeper_as_a_Shared_storage_EXPERIMENTAL)
Purpose
-------
@@ -572,116 +571,3 @@ Automatic Failover FAQ
using the same `hdfs haadmin` command. It will perform a coordinated
failover.
-BookKeeper as a Shared storage (EXPERIMENTAL)
----------------------------------------------
-
-One option for shared storage for the NameNode is BookKeeper. BookKeeper achieves high availability and strong durability guarantees by replicating edit log entries across multiple storage nodes. The edit log can be striped across the storage nodes for high performance. Fencing is supported in the protocol, i.e, BookKeeper will not allow two writers to write the single edit log.
-
-The meta data for BookKeeper is stored in ZooKeeper. In current HA architecture, a Zookeeper cluster is required for ZKFC. The same cluster can be for BookKeeper metadata.
-
-For more details on building a BookKeeper cluster, please refer to the [BookKeeper documentation](http://zookeeper.apache.org/bookkeeper/docs/trunk/bookkeeperConfig.html )
-
-The BookKeeperJournalManager is an implementation of the HDFS JournalManager interface, which allows custom write ahead logging implementations to be plugged into the HDFS NameNode.
-
-* **BookKeeper Journal Manager**
-
- To use BookKeeperJournalManager, add the following to hdfs-site.xml.
-
- <property>
- <name>dfs.namenode.shared.edits.dir</name>
- <value>bookkeeper://zk1:2181;zk2:2181;zk3:2181/hdfsjournal</value>
- </property>
-
- <property>
- <name>dfs.namenode.edits.journal-plugin.bookkeeper</name>
- <value>org.apache.hadoop.contrib.bkjournal.BookKeeperJournalManager</value>
- </property>
-
- The URI format for bookkeeper is `bookkeeper://[zkEnsemble]/[rootZnode] [zookkeeper ensemble]`
- is a list of semi-colon separated, zookeeper host:port
- pairs. In the example above there are 3 servers, in the ensemble,
- zk1, zk2 & zk3, each one listening on port 2181.
-
- `[root znode]` is the path of the zookeeper znode, under which the edit log
- information will be stored.
-
- The class specified for the journal-plugin must be available in the NameNode's
- classpath. We explain how to generate a jar file with the journal manager and
- its dependencies, and how to put it into the classpath below.
-
-* **More configuration options**
-
- * **dfs.namenode.bookkeeperjournal.output-buffer-size** -
- Number of bytes a bookkeeper journal stream will buffer before
- forcing a flush. Default is 1024.
-
- <property>
- <name>dfs.namenode.bookkeeperjournal.output-buffer-size</name>
- <value>1024</value>
- </property>
-
- * **dfs.namenode.bookkeeperjournal.ensemble-size** -
- Number of bookkeeper servers in edit log ensembles. This
- is the number of bookkeeper servers which need to be available
- for the edit log to be writable. Default is 3.
-
- <property>
- <name>dfs.namenode.bookkeeperjournal.ensemble-size</name>
- <value>3</value>
- </property>
-
- * **dfs.namenode.bookkeeperjournal.quorum-size** -
- Number of bookkeeper servers in the write quorum. This is the
- number of bookkeeper servers which must have acknowledged the
- write of an entry before it is considered written. Default is 2.
-
- <property>
- <name>dfs.namenode.bookkeeperjournal.quorum-size</name>
- <value>2</value>
- </property>
-
- * **dfs.namenode.bookkeeperjournal.digestPw** -
- Password to use when creating edit log segments.
-
- <property>
- <name>dfs.namenode.bookkeeperjournal.digestPw</name>
- <value>myPassword</value>
- </property>
-
- * **dfs.namenode.bookkeeperjournal.zk.session.timeout** -
- Session timeout for Zookeeper client from BookKeeper Journal Manager.
- Hadoop recommends that this value should be less than the ZKFC
- session timeout value. Default value is 3000.
-
- <property>
- <name>dfs.namenode.bookkeeperjournal.zk.session.timeout</name>
- <value>3000</value>
- </property>
-
-* **Building BookKeeper Journal Manager plugin jar**
-
- To generate the distribution packages for BK journal, do the following.
-
- $ mvn clean package -Pdist
-
- This will generate a jar with the BookKeeperJournalManager,
- hadoop-hdfs/src/contrib/bkjournal/target/hadoop-hdfs-bkjournal-*VERSION*.jar
-
- Note that the -Pdist part of the build command is important, this would
- copy the dependent bookkeeper-server jar under
- hadoop-hdfs/src/contrib/bkjournal/target/lib.
-
-* **Putting the BookKeeperJournalManager in the NameNode classpath**
-
- To run a HDFS namenode using BookKeeper as a backend, copy the bkjournal and
- bookkeeper-server jar, mentioned above, into the lib directory of hdfs. In the
- standard distribution of HDFS, this is at $HADOOP\_HDFS\_HOME/share/hadoop/hdfs/lib/
-
- cp hadoop-hdfs/src/contrib/bkjournal/target/hadoop-hdfs-bkjournal-*VERSION*.jar $HADOOP\_HDFS\_HOME/share/hadoop/hdfs/lib/
-
-* **Current limitations**
-
- 1) Security in BookKeeper. BookKeeper does not support SASL nor SSL for
- connections between the NameNode and BookKeeper storage nodes.
-
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31195488/hadoop-hdfs-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml
index 4bb0545..a0993c9 100644
--- a/hadoop-hdfs-project/pom.xml
+++ b/hadoop-hdfs-project/pom.xml
@@ -35,7 +35,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<module>hadoop-hdfs-client</module>
<module>hadoop-hdfs-native-client</module>
<module>hadoop-hdfs-httpfs</module>
- <module>hadoop-hdfs/src/contrib/bkjournal</module>
<module>hadoop-hdfs-nfs</module>
</modules>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31195488/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 26ac16c..58abaf3 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -930,12 +930,6 @@
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.bookkeeper</groupId>
- <artifactId>bookkeeper-server</artifactId>
- <version>4.2.3</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>org.hsqldb</groupId>
<artifactId>hsqldb</artifactId>
<version>2.0.0</version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org